diff --git a/.github/scripts/changed_apps.py b/.github/scripts/changed_apps.py index 46e551a1e4..e2b64970d4 100755 --- a/.github/scripts/changed_apps.py +++ b/.github/scripts/changed_apps.py @@ -62,6 +62,10 @@ def find_test_files(changed_files): print("Skipped apps based on the EXCLUDE_TESTS list:", file=sys.stderr) print("\n".join(skipped), file=sys.stderr) + if len(matrix) > 256: + print(f"Github Actions has a limit of 256 matrix jobs. Cannot run [{len(matrix)}] tests.", file=sys.stderr) + sys.exit(1) + return matrix diff --git a/ix-dev/community/actual-budget/app.yaml b/ix-dev/community/actual-budget/app.yaml index 991bbf7ef7..d9b3d50493 100644 --- a/ix-dev/community/actual-budget/app.yaml +++ b/ix-dev/community/actual-budget/app.yaml @@ -10,8 +10,8 @@ icon: https://media.sys.truenas.net/apps/actual-budget/icons/icon.png keywords: - finance - budget -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -33,4 +33,4 @@ sources: - https://hub.docker.com/r/actualbudget/actual-server title: Actual Budget train: community -version: 1.2.1 +version: 1.2.2 diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/container.py b/ix-dev/community/actual-budget/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/deps.py b/ix-dev/community/actual-budget/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/devices.py b/ix-dev/community/actual-budget/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/environment.py b/ix-dev/community/actual-budget/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/functions.py b/ix-dev/community/actual-budget/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/actual-budget/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/storage.py b/ix-dev/community/actual-budget/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/validations.py b/ix-dev/community/actual-budget/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/configs.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_1_0/container.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/depends.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_1_0/deps.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/device.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/device.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_1_0/devices.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/dns.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_1_0/environment.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/error.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/error.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/functions.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/functions.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/functions.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/functions.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/labels.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/notes.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/portal.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/portals.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/ports.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/render.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/render.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/resources.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/restart.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/storage.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_functions.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_functions.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_functions.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_functions.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_1_0/validations.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/actual-budget/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/actual-budget/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/actual-budget/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/actual-budget/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/adguard-home/app.yaml b/ix-dev/community/adguard-home/app.yaml index e22c53e627..aaf6d3d77b 100644 --- a/ix-dev/community/adguard-home/app.yaml +++ b/ix-dev/community/adguard-home/app.yaml @@ -20,8 +20,8 @@ icon: https://media.sys.truenas.net/apps/adguard-home/icons/icon.svg keywords: - dns - adblock -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -42,4 +42,4 @@ sources: - https://hub.docker.com/r/adguard/adguardhome title: AdGuard Home train: community -version: 1.1.2 +version: 1.1.3 diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/container.py b/ix-dev/community/adguard-home/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/deps.py b/ix-dev/community/adguard-home/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/devices.py b/ix-dev/community/adguard-home/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/environment.py b/ix-dev/community/adguard-home/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/functions.py b/ix-dev/community/adguard-home/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/adguard-home/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/storage.py b/ix-dev/community/adguard-home/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/validations.py b/ix-dev/community/adguard-home/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/configs.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_1_0/container.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/depends.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_1_0/deps.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/device.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/device.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_1_0/devices.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/dns.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_1_0/environment.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/error.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/error.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_1_0/functions.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/labels.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/notes.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/portal.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/portals.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/ports.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/render.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/render.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/resources.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/restart.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/storage.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/storage.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/sysctls.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/sysctls.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_functions.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_functions.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_functions.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_functions.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_sysctls.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_sysctls.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_volumes.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_volumes.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_1_0/validations.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/adguard-home/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/adguard-home/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/adguard-home/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/adguard-home/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/audiobookshelf/app.yaml b/ix-dev/community/audiobookshelf/app.yaml index 7e7af41d77..30422bdd2b 100644 --- a/ix-dev/community/audiobookshelf/app.yaml +++ b/ix-dev/community/audiobookshelf/app.yaml @@ -9,8 +9,8 @@ icon: https://media.sys.truenas.net/apps/audiobookshelf/icons/icon.svg keywords: - media - audiobook -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -33,4 +33,4 @@ sources: - https://github.com/advplyr/audiobookshelf title: Audiobookshelf train: community -version: 1.3.2 +version: 1.3.3 diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/container.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/deps.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/devices.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/environment.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/functions.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/storage.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/validations.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/configs.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/container.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/depends.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/deps.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/device.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/device.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/devices.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/dns.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/environment.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/error.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/error.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/functions.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/labels.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/notes.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/portal.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/portals.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/ports.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/render.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/render.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/resources.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/restart.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/storage.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/storage.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/sysctls.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/sysctls.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_functions.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_functions.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_functions.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_functions.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_sysctls.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_sysctls.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_volumes.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_volumes.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/validations.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/audiobookshelf/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/audiobookshelf/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/autobrr/app.yaml b/ix-dev/community/autobrr/app.yaml index 6c0369bc77..88a5281cbb 100644 --- a/ix-dev/community/autobrr/app.yaml +++ b/ix-dev/community/autobrr/app.yaml @@ -10,8 +10,8 @@ keywords: - media - torrent - usenet -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -31,4 +31,4 @@ sources: - https://github.com/autobrr/autobrr title: Autobrr train: community -version: 1.2.1 +version: 1.2.2 diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/container.py b/ix-dev/community/autobrr/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/autobrr/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/deps.py b/ix-dev/community/autobrr/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/autobrr/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/devices.py b/ix-dev/community/autobrr/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/autobrr/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/environment.py b/ix-dev/community/autobrr/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/autobrr/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/functions.py b/ix-dev/community/autobrr/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/autobrr/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/autobrr/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/autobrr/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/storage.py b/ix-dev/community/autobrr/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/autobrr/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/validations.py b/ix-dev/community/autobrr/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/autobrr/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/configs.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_1_0/container.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/autobrr/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/depends.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_1_0/deps.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/autobrr/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/autobrr/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/autobrr/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/autobrr/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/autobrr/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/autobrr/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/autobrr/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/autobrr/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/autobrr/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/device.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/device.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_1_0/devices.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/autobrr/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/dns.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_1_0/environment.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/autobrr/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/error.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/error.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_1_0/functions.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/autobrr/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/autobrr/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/autobrr/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/labels.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/notes.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/portal.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/portals.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/ports.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/render.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/render.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/resources.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/restart.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/storage.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/storage.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/sysctls.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/sysctls.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_sysctls.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_sysctls.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_volumes.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_volumes.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_1_0/validations.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/autobrr/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/autobrr/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/autobrr/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/autobrr/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/autobrr/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/bazarr/app.yaml b/ix-dev/community/bazarr/app.yaml index ae3505cda1..deb898fc22 100644 --- a/ix-dev/community/bazarr/app.yaml +++ b/ix-dev/community/bazarr/app.yaml @@ -10,8 +10,8 @@ icon: https://media.sys.truenas.net/apps/bazarr/icons/icon.png keywords: - media - subtitles -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -31,4 +31,4 @@ sources: - https://github.com/morpheus65535/bazarr title: Bazarr train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/container.py b/ix-dev/community/bazarr/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/bazarr/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/deps.py b/ix-dev/community/bazarr/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/bazarr/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/devices.py b/ix-dev/community/bazarr/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/bazarr/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/environment.py b/ix-dev/community/bazarr/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/bazarr/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/functions.py b/ix-dev/community/bazarr/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/bazarr/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/bazarr/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/bazarr/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/storage.py b/ix-dev/community/bazarr/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/bazarr/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/validations.py b/ix-dev/community/bazarr/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/bazarr/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/configs.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_1_0/container.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/bazarr/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/depends.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_1_0/deps.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/bazarr/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/bazarr/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/bazarr/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/bazarr/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/bazarr/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/bazarr/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/bazarr/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/bazarr/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/bazarr/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/device.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/device.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_1_0/devices.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/bazarr/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/dns.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_1_0/environment.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/bazarr/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/error.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/error.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_1_0/functions.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/bazarr/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/bazarr/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/bazarr/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/labels.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/notes.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/portal.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/portals.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/ports.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/render.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/render.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/resources.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/restart.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/storage.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/storage.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/sysctls.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/sysctls.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_sysctls.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_sysctls.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_volumes.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_volumes.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_1_0/validations.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/bazarr/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/bazarr/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/bazarr/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/bazarr/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/bazarr/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/briefkasten/app.yaml b/ix-dev/community/briefkasten/app.yaml index f10bec3e2f..75900b180f 100644 --- a/ix-dev/community/briefkasten/app.yaml +++ b/ix-dev/community/briefkasten/app.yaml @@ -8,8 +8,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/briefkasten/icons/icon.svg keywords: - bookmark -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -35,4 +35,4 @@ sources: - https://docs.briefkastenhq.com/ title: Briefkasten train: community -version: 1.1.3 +version: 1.1.4 diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/container.py b/ix-dev/community/briefkasten/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/deps.py b/ix-dev/community/briefkasten/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/devices.py b/ix-dev/community/briefkasten/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/environment.py b/ix-dev/community/briefkasten/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/functions.py b/ix-dev/community/briefkasten/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/briefkasten/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/storage.py b/ix-dev/community/briefkasten/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/validations.py b/ix-dev/community/briefkasten/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/configs.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_1_0/container.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/depends.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_1_0/deps.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/device.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/device.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_1_0/devices.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/dns.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_1_0/environment.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/error.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/error.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_1_0/functions.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/labels.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/notes.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/portal.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/portals.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/ports.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/render.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/render.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/resources.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/restart.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/storage.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/storage.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/sysctls.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/sysctls.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_sysctls.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_sysctls.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_volumes.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_volumes.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_1_0/validations.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/briefkasten/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/briefkasten/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/briefkasten/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/briefkasten/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/calibre-web/app.yaml b/ix-dev/community/calibre-web/app.yaml index 66e4d4f055..dfe549a74c 100644 --- a/ix-dev/community/calibre-web/app.yaml +++ b/ix-dev/community/calibre-web/app.yaml @@ -10,8 +10,8 @@ icon: https://media.sys.truenas.net/apps/calibre-web/icons/icon.svg keywords: - media - ebooks -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -30,4 +30,4 @@ sources: - https://github.com/janeczku/calibre-web title: Calibre Web train: community -version: 1.0.3 +version: 1.0.4 diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/container.py b/ix-dev/community/calibre-web/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/deps.py b/ix-dev/community/calibre-web/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/devices.py b/ix-dev/community/calibre-web/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/environment.py b/ix-dev/community/calibre-web/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/functions.py b/ix-dev/community/calibre-web/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/calibre-web/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/storage.py b/ix-dev/community/calibre-web/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/validations.py b/ix-dev/community/calibre-web/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/configs.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_1_0/container.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/depends.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_1_0/deps.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/device.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/device.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_1_0/devices.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/dns.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_1_0/environment.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/error.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/error.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_1_0/functions.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/labels.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/notes.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/portal.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/portals.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/ports.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/render.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/render.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/resources.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/restart.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/storage.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/storage.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/sysctls.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/sysctls.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_sysctls.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_sysctls.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_volumes.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_volumes.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_1_0/validations.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/calibre-web/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/calibre-web/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/calibre-web/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/calibre-web/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/calibre/app.yaml b/ix-dev/community/calibre/app.yaml index d25ce3bfd3..d627b23152 100644 --- a/ix-dev/community/calibre/app.yaml +++ b/ix-dev/community/calibre/app.yaml @@ -19,8 +19,8 @@ icon: https://media.sys.truenas.net/apps/calibre/icons/icon.png keywords: - media - ebooks -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -45,4 +45,4 @@ sources: - https://calibre-ebook.com/ title: Calibre train: community -version: 1.0.3 +version: 1.0.4 diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/container.py b/ix-dev/community/calibre/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/calibre/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/deps.py b/ix-dev/community/calibre/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/calibre/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/devices.py b/ix-dev/community/calibre/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/calibre/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/environment.py b/ix-dev/community/calibre/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/calibre/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/functions.py b/ix-dev/community/calibre/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/calibre/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/calibre/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/calibre/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/storage.py b/ix-dev/community/calibre/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/calibre/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/validations.py b/ix-dev/community/calibre/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/calibre/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/configs.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_1_0/container.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/calibre/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/depends.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_1_0/deps.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/calibre/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/calibre/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/calibre/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/calibre/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/calibre/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/calibre/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/calibre/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/calibre/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/calibre/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/device.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/device.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_1_0/devices.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/calibre/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/dns.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_1_0/environment.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/calibre/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/error.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/error.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_1_0/functions.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/calibre/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/calibre/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/calibre/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/labels.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/notes.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/portal.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/portals.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/ports.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/render.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/render.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/resources.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/restart.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/storage.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/storage.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/sysctls.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/sysctls.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_sysctls.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_sysctls.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_volumes.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_volumes.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_1_0/validations.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/calibre/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/calibre/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/calibre/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/calibre/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/calibre/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/castopod/app.yaml b/ix-dev/community/castopod/app.yaml index a6ec26f11b..2615bcb1ae 100644 --- a/ix-dev/community/castopod/app.yaml +++ b/ix-dev/community/castopod/app.yaml @@ -19,8 +19,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/castopod/icons/icon.svg keywords: - podcast -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -49,4 +49,4 @@ sources: - https://code.castopod.org/adaures/castopod title: Castopod train: community -version: 1.1.2 +version: 1.1.3 diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/container.py b/ix-dev/community/castopod/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/castopod/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/deps.py b/ix-dev/community/castopod/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/castopod/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/devices.py b/ix-dev/community/castopod/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/castopod/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/environment.py b/ix-dev/community/castopod/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/castopod/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/functions.py b/ix-dev/community/castopod/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/castopod/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/castopod/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/castopod/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/validations.py b/ix-dev/community/castopod/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/castopod/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/configs.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_1_0/container.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/castopod/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/depends.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_1_0/deps.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/castopod/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/castopod/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/castopod/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/castopod/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/castopod/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/castopod/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/castopod/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/castopod/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/castopod/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/device.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/device.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_1_0/devices.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/castopod/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/dns.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_1_0/environment.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/castopod/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/error.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/error.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_1_0/functions.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/castopod/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/castopod/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/castopod/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/labels.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/notes.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/portal.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/portals.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/ports.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/render.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/render.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/resources.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/restart.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/storage.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/storage.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/sysctls.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/sysctls.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_sysctls.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_sysctls.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_volumes.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_volumes.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_1_0/validations.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/castopod/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/castopod/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/castopod/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/castopod/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/castopod/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/chia/app.yaml b/ix-dev/community/chia/app.yaml index 826b1a3d1b..69f6184d3f 100644 --- a/ix-dev/community/chia/app.yaml +++ b/ix-dev/community/chia/app.yaml @@ -11,8 +11,8 @@ keywords: - blockchain - hard-drive - chia -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -30,4 +30,4 @@ sources: - https://www.chia.net/ title: Chia train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/container.py b/ix-dev/community/chia/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/chia/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/deps.py b/ix-dev/community/chia/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/chia/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/devices.py b/ix-dev/community/chia/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/chia/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/environment.py b/ix-dev/community/chia/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/chia/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/functions.py b/ix-dev/community/chia/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/chia/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/chia/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/chia/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/storage.py b/ix-dev/community/chia/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/chia/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/validations.py b/ix-dev/community/chia/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/chia/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/chia/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/configs.py b/ix-dev/community/chia/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/chia/templates/library/base_v2_1_0/container.py b/ix-dev/community/chia/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/chia/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/depends.py b/ix-dev/community/chia/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/chia/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/chia/templates/library/base_v2_1_0/deps.py b/ix-dev/community/chia/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/chia/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/chia/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/chia/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/chia/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/chia/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/chia/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/chia/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/chia/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/chia/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/chia/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/chia/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/chia/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/chia/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/device.py b/ix-dev/community/chia/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/device.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/chia/templates/library/base_v2_1_0/devices.py b/ix-dev/community/chia/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/chia/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/dns.py b/ix-dev/community/chia/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/chia/templates/library/base_v2_1_0/environment.py b/ix-dev/community/chia/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/chia/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/error.py b/ix-dev/community/chia/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/error.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/chia/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/chia/templates/library/base_v2_1_0/functions.py b/ix-dev/community/chia/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/chia/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/chia/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/chia/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/chia/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/labels.py b/ix-dev/community/chia/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/notes.py b/ix-dev/community/chia/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/portal.py b/ix-dev/community/chia/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/portals.py b/ix-dev/community/chia/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/ports.py b/ix-dev/community/chia/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/render.py b/ix-dev/community/chia/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/render.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/resources.py b/ix-dev/community/chia/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/restart.py b/ix-dev/community/chia/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/storage.py b/ix-dev/community/chia/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/storage.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/sysctls.py b/ix-dev/community/chia/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/sysctls.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_sysctls.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_sysctls.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_volumes.py b/ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_volumes.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/chia/templates/library/base_v2_1_0/validations.py b/ix-dev/community/chia/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/chia/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/chia/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/chia/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/chia/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/chia/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/chia/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/chia/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/chia/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/chia/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/clamav/app.yaml b/ix-dev/community/clamav/app.yaml index cc241eda9d..01761d87fb 100644 --- a/ix-dev/community/clamav/app.yaml +++ b/ix-dev/community/clamav/app.yaml @@ -19,8 +19,8 @@ icon: https://media.sys.truenas.net/apps/clamav/icons/icon.png keywords: - anti-virus - clamav -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -38,4 +38,4 @@ sources: - https://www.clamav.net/ title: ClamAV train: community -version: 1.2.6 +version: 1.2.7 diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/container.py b/ix-dev/community/clamav/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/clamav/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/deps.py b/ix-dev/community/clamav/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/clamav/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/devices.py b/ix-dev/community/clamav/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/clamav/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/environment.py b/ix-dev/community/clamav/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/clamav/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/functions.py b/ix-dev/community/clamav/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/clamav/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/clamav/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/clamav/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/storage.py b/ix-dev/community/clamav/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/clamav/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/validations.py b/ix-dev/community/clamav/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/clamav/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/configs.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_1_0/container.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/clamav/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/depends.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_1_0/deps.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/clamav/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/clamav/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/clamav/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/clamav/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/clamav/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/clamav/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/clamav/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/clamav/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/clamav/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/device.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/device.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_1_0/devices.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/clamav/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/dns.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_1_0/environment.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/clamav/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/error.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/error.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_1_0/functions.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/clamav/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/clamav/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/clamav/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/labels.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/notes.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/portal.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/portals.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/ports.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/render.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/render.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/resources.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/restart.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/storage.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/storage.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/sysctls.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/sysctls.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_sysctls.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_sysctls.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_volumes.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_volumes.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_1_0/validations.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/clamav/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/clamav/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/clamav/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/clamav/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/clamav/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/cloudflared/app.yaml b/ix-dev/community/cloudflared/app.yaml index c4b25c4b33..27207d64cc 100644 --- a/ix-dev/community/cloudflared/app.yaml +++ b/ix-dev/community/cloudflared/app.yaml @@ -11,8 +11,8 @@ keywords: - network - cloudflare - tunnel -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -30,4 +30,4 @@ sources: - https://hub.docker.com/r/cloudflare/cloudflared title: Cloudflared train: community -version: 1.2.0 +version: 1.2.1 diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/container.py b/ix-dev/community/cloudflared/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/deps.py b/ix-dev/community/cloudflared/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/devices.py b/ix-dev/community/cloudflared/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/environment.py b/ix-dev/community/cloudflared/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/functions.py b/ix-dev/community/cloudflared/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/cloudflared/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/storage.py b/ix-dev/community/cloudflared/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/validations.py b/ix-dev/community/cloudflared/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/configs.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_1_0/container.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/depends.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_1_0/deps.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/device.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/device.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_1_0/devices.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/dns.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_1_0/environment.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/error.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/error.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_1_0/functions.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/labels.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/notes.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/portal.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/portals.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/ports.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/render.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/render.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/resources.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/restart.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/storage.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/storage.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_volumes.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_volumes.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_1_0/validations.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/cloudflared/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/cloudflared/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/cloudflared/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/cloudflared/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/dashy/app.yaml b/ix-dev/community/dashy/app.yaml index 60adc12977..bd04149d61 100644 --- a/ix-dev/community/dashy/app.yaml +++ b/ix-dev/community/dashy/app.yaml @@ -8,8 +8,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/dashy/icons/icon.png keywords: - dashboard -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -29,4 +29,4 @@ sources: - https://github.com/lissy93/dashy title: Dashy train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/container.py b/ix-dev/community/dashy/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/dashy/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/deps.py b/ix-dev/community/dashy/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/dashy/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/devices.py b/ix-dev/community/dashy/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/dashy/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/environment.py b/ix-dev/community/dashy/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/dashy/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/functions.py b/ix-dev/community/dashy/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/dashy/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/dashy/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/dashy/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/storage.py b/ix-dev/community/dashy/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/dashy/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/validations.py b/ix-dev/community/dashy/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/dashy/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/configs.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_1_0/container.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/dashy/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/depends.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_1_0/deps.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/dashy/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/dashy/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/dashy/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/dashy/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/dashy/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/dashy/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/dashy/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/dashy/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/dashy/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/device.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/device.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_1_0/devices.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/dashy/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/dns.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_1_0/environment.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/dashy/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/error.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/error.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_1_0/functions.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/dashy/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/dashy/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/dashy/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/labels.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/notes.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/portal.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/portals.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/ports.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/render.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/render.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/resources.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/restart.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/storage.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/storage.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/sysctls.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/sysctls.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_sysctls.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_sysctls.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_volumes.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_volumes.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_1_0/validations.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/dashy/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/dashy/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/dashy/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/dashy/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/dashy/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/ddns-updater/app.yaml b/ix-dev/community/ddns-updater/app.yaml index 8485141684..35b7422472 100644 --- a/ix-dev/community/ddns-updater/app.yaml +++ b/ix-dev/community/ddns-updater/app.yaml @@ -9,8 +9,8 @@ icon: https://media.sys.truenas.net/apps/ddns-updater/icons/icon.svg keywords: - ddns-updater - ddns -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -30,4 +30,4 @@ sources: - https://hub.docker.com/r/qmcgaw/ddns-updater title: DDNS Updater train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/container.py b/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/deps.py b/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/devices.py b/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/environment.py b/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/functions.py b/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/storage.py b/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/validations.py b/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/configs.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/container.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/depends.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/deps.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/device.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/device.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/devices.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/dns.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/environment.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/error.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/error.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/functions.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/labels.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/notes.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/portal.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/portals.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/ports.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/render.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/render.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/resources.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/restart.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/storage.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/storage.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_volumes.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_volumes.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/validations.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/ddns-updater/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/ddns-updater/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/ddns-updater/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/ddns-updater/templates/library/community/ddns-updater/v1_1_0/__init__.py b/ix-dev/community/ddns-updater/templates/library/community/ddns-updater/v1_1_1/__init__.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/community/ddns-updater/v1_1_0/__init__.py rename to ix-dev/community/ddns-updater/templates/library/community/ddns-updater/v1_1_1/__init__.py diff --git a/ix-dev/community/ddns-updater/templates/library/community/ddns-updater/v1_1_0/config.py b/ix-dev/community/ddns-updater/templates/library/community/ddns-updater/v1_1_1/config.py similarity index 100% rename from ix-dev/community/ddns-updater/templates/library/community/ddns-updater/v1_1_0/config.py rename to ix-dev/community/ddns-updater/templates/library/community/ddns-updater/v1_1_1/config.py diff --git a/ix-dev/community/deluge/app.yaml b/ix-dev/community/deluge/app.yaml index 1446bf44e5..011259e0df 100644 --- a/ix-dev/community/deluge/app.yaml +++ b/ix-dev/community/deluge/app.yaml @@ -19,8 +19,8 @@ icon: https://media.sys.truenas.net/apps/deluge/icons/icon.png keywords: - torrent - download -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -38,4 +38,4 @@ sources: - https://deluge-torrent.org/ title: Deluge train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/container.py b/ix-dev/community/deluge/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/deluge/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/deps.py b/ix-dev/community/deluge/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/deluge/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/devices.py b/ix-dev/community/deluge/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/deluge/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/environment.py b/ix-dev/community/deluge/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/deluge/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/functions.py b/ix-dev/community/deluge/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/deluge/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/deluge/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/deluge/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/storage.py b/ix-dev/community/deluge/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/deluge/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/validations.py b/ix-dev/community/deluge/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/deluge/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/configs.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_1_0/container.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/deluge/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/depends.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_1_0/deps.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/deluge/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/deluge/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/deluge/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/deluge/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/deluge/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/deluge/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/deluge/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/deluge/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/deluge/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/device.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/device.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_1_0/devices.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/deluge/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/dns.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_1_0/environment.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/deluge/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/error.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/error.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_1_0/functions.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/deluge/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/deluge/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/deluge/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/labels.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/notes.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/portal.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/portals.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/ports.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/render.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/render.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/resources.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/restart.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/storage.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/storage.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_volumes.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_volumes.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_1_0/validations.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/deluge/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/deluge/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/deluge/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/deluge/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/deluge/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/distribution/app.yaml b/ix-dev/community/distribution/app.yaml index 33ea52871f..bbd0b5c7d9 100644 --- a/ix-dev/community/distribution/app.yaml +++ b/ix-dev/community/distribution/app.yaml @@ -10,8 +10,8 @@ icon: https://media.sys.truenas.net/apps/distribution/icons/icon.svg keywords: - registry - container -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -30,4 +30,4 @@ sources: - https://github.com/distribution/distribution title: Distribution train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/container.py b/ix-dev/community/distribution/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/distribution/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/deps.py b/ix-dev/community/distribution/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/distribution/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/devices.py b/ix-dev/community/distribution/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/distribution/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/environment.py b/ix-dev/community/distribution/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/distribution/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/functions.py b/ix-dev/community/distribution/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/distribution/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/distribution/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/distribution/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/storage.py b/ix-dev/community/distribution/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/distribution/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/validations.py b/ix-dev/community/distribution/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/distribution/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/configs.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_1_0/container.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/distribution/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/depends.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_1_0/deps.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/distribution/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/distribution/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/distribution/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/distribution/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/distribution/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/distribution/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/distribution/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/distribution/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/distribution/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/device.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/device.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_1_0/devices.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/distribution/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/dns.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_1_0/environment.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/distribution/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/error.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/error.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_1_0/functions.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/distribution/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/distribution/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/distribution/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/labels.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/notes.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/portal.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/portals.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/ports.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/render.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/render.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/resources.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/restart.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/storage.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_1_0/validations.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/distribution/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/distribution/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/distribution/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/distribution/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/distribution/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/dockge/app.yaml b/ix-dev/community/dockge/app.yaml index 7359f08c9c..e163510a60 100644 --- a/ix-dev/community/dockge/app.yaml +++ b/ix-dev/community/dockge/app.yaml @@ -28,8 +28,8 @@ icon: https://media.sys.truenas.net/apps/dockge/icons/icon.svg keywords: - docker - compose -lib_version: 2.0.23 -lib_version_hash: 8909de0f230ddb07f219f65eb8e67f9b8a112e9c310bee801834248854aa91af +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -48,4 +48,4 @@ sources: - https://github.com/louislam/dockge title: Dockge train: community -version: 1.1.1 +version: 1.1.2 diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/container.py b/ix-dev/community/dockge/templates/library/base_v2_0_23/container.py deleted file mode 100644 index 34df9ec83b..0000000000 --- a/ix-dev/community/dockge/templates/library/base_v2_0_23/container.py +++ /dev/null @@ -1,304 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/deps.py b/ix-dev/community/dockge/templates/library/base_v2_0_23/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/dockge/templates/library/base_v2_0_23/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/devices.py b/ix-dev/community/dockge/templates/library/base_v2_0_23/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/dockge/templates/library/base_v2_0_23/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/environment.py b/ix-dev/community/dockge/templates/library/base_v2_0_23/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/dockge/templates/library/base_v2_0_23/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/functions.py b/ix-dev/community/dockge/templates/library/base_v2_0_23/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/dockge/templates/library/base_v2_0_23/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/healthcheck.py b/ix-dev/community/dockge/templates/library/base_v2_0_23/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/dockge/templates/library/base_v2_0_23/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/storage.py b/ix-dev/community/dockge/templates/library/base_v2_0_23/storage.py deleted file mode 100644 index bf39d45daa..0000000000 --- a/ix-dev/community/dockge/templates/library/base_v2_0_23/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def _add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_container.py b/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_container.py deleted file mode 100644 index 5e51b86198..0000000000 --- a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_container.py +++ /dev/null @@ -1,294 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_deps.py b/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_device.py b/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_environment.py b/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_functions.py b/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_healthcheck.py b/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_volumes.py b/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_volumes.py deleted file mode 100644 index 2084de09e0..0000000000 --- a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage._add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage._add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage._add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/validations.py b/ix-dev/community/dockge/templates/library/base_v2_0_23/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/dockge/templates/library/base_v2_0_23/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/__init__.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/__init__.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/configs.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/configs.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_1_0/container.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/dockge/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/depends.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/depends.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/deploy.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/deploy.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_1_0/deps.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/dockge/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/dockge/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/dockge/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/dockge/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/dockge/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/dockge/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/dockge/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/dockge/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/dockge/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/device.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/device.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_1_0/devices.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/dockge/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/dns.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/dns.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_1_0/environment.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/dockge/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/error.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/error.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/formatter.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/formatter.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_1_0/functions.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/dockge/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/dockge/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/dockge/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/labels.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/labels.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/notes.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/notes.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/portal.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/portal.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/portals.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/portals.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/ports.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/ports.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/render.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/render.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/resources.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/resources.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/restart.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/restart.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/storage.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/storage.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/__init__.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/tests/__init__.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_build_image.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_build_image.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_configs.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_configs.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_depends.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_depends.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_dns.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_dns.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_formatter.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_formatter.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_labels.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_labels.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_notes.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_notes.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_portal.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_portal.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_ports.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_ports.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_render.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_render.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_resources.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_resources.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_restart.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/tests/test_restart.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_volumes.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_volumes.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_1_0/validations.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/dockge/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/volume_mount.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/volume_mount.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/volume_mount_types.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/volume_mount_types.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/volume_sources.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/volume_sources.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/volume_types.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/volume_types.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/dockge/templates/library/base_v2_0_23/volumes.py b/ix-dev/community/dockge/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/dockge/templates/library/base_v2_0_23/volumes.py rename to ix-dev/community/dockge/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/drawio/app.yaml b/ix-dev/community/drawio/app.yaml index 0985cb52b8..8814dc0689 100644 --- a/ix-dev/community/drawio/app.yaml +++ b/ix-dev/community/drawio/app.yaml @@ -9,8 +9,8 @@ icon: https://media.sys.truenas.net/apps/drawio/icons/icon.png keywords: - diagram - whiteboard -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -32,4 +32,4 @@ sources: - https://github.com/jgraph/drawio title: Draw.io train: community -version: 1.2.0 +version: 1.2.1 diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/container.py b/ix-dev/community/drawio/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/drawio/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/deps.py b/ix-dev/community/drawio/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/drawio/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/devices.py b/ix-dev/community/drawio/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/drawio/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/environment.py b/ix-dev/community/drawio/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/drawio/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/functions.py b/ix-dev/community/drawio/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/drawio/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/drawio/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/drawio/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/storage.py b/ix-dev/community/drawio/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/drawio/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/validations.py b/ix-dev/community/drawio/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/drawio/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/configs.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_1_0/container.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/drawio/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/depends.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_1_0/deps.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/drawio/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/drawio/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/drawio/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/drawio/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/drawio/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/drawio/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/drawio/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/drawio/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/drawio/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/device.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/device.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_1_0/devices.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/drawio/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/dns.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_1_0/environment.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/drawio/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/error.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/error.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_1_0/functions.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/drawio/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/drawio/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/drawio/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/labels.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/notes.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/portal.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/portals.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/ports.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/render.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/render.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/resources.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/restart.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/storage.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/sysctls.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/sysctls.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_sysctls.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_sysctls.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_1_0/validations.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/drawio/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/drawio/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/drawio/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/drawio/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/drawio/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/eclipse-mosquitto/app.yaml b/ix-dev/community/eclipse-mosquitto/app.yaml index 30051715b8..fce321359f 100644 --- a/ix-dev/community/eclipse-mosquitto/app.yaml +++ b/ix-dev/community/eclipse-mosquitto/app.yaml @@ -9,8 +9,8 @@ icon: https://media.sys.truenas.net/apps/eclipse-mosquitto/icons/icon.svg keywords: - networking - mqtt -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -28,4 +28,4 @@ sources: - https://hub.docker.com/_/eclipse-mosquitto title: Eclipse Mosquitto train: community -version: 1.0.2 +version: 1.0.3 diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/container.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/deps.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/devices.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/environment.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/functions.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/storage.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/validations.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/configs.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/container.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/depends.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/deps.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/device.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/device.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/devices.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/dns.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/environment.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/error.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/error.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/functions.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/labels.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/notes.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/portal.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/portals.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/ports.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/render.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/render.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/resources.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/restart.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/storage.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/validations.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/eclipse-mosquitto/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/eclipse-mosquitto/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/filebrowser/app.yaml b/ix-dev/community/filebrowser/app.yaml index 9563278285..d338c23d6d 100644 --- a/ix-dev/community/filebrowser/app.yaml +++ b/ix-dev/community/filebrowser/app.yaml @@ -10,8 +10,8 @@ icon: https://media.sys.truenas.net/apps/filebrowser/icons/icon.png keywords: - files - browser -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -33,4 +33,4 @@ sources: - https://hub.docker.com/r/filebrowser/filebrowser title: File Browser train: community -version: 1.2.0 +version: 1.2.1 diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/container.py b/ix-dev/community/filebrowser/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/deps.py b/ix-dev/community/filebrowser/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/devices.py b/ix-dev/community/filebrowser/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/environment.py b/ix-dev/community/filebrowser/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/functions.py b/ix-dev/community/filebrowser/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/filebrowser/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/storage.py b/ix-dev/community/filebrowser/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/validations.py b/ix-dev/community/filebrowser/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/configs.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_1_0/container.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/depends.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_1_0/deps.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/device.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/device.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_1_0/devices.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/dns.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_1_0/environment.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/error.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/error.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_1_0/functions.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/labels.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/notes.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/portal.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/portals.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/ports.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/render.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/render.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/resources.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/restart.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/storage.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_1_0/validations.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/filebrowser/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/filebrowser/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/filebrowser/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/filebrowser/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/filestash/app.yaml b/ix-dev/community/filestash/app.yaml index c72657e974..856650088c 100644 --- a/ix-dev/community/filestash/app.yaml +++ b/ix-dev/community/filestash/app.yaml @@ -10,8 +10,8 @@ icon: https://media.sys.truenas.net/apps/filestash/icons/icon.svg keywords: - storage - file manager -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -32,4 +32,4 @@ sources: - https://github.com/mickael-kerjean/filestash title: Filestash train: community -version: 1.0.4 +version: 1.0.5 diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/container.py b/ix-dev/community/filestash/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/filestash/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/deps.py b/ix-dev/community/filestash/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/filestash/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/devices.py b/ix-dev/community/filestash/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/filestash/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/environment.py b/ix-dev/community/filestash/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/filestash/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/functions.py b/ix-dev/community/filestash/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/filestash/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/filestash/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/filestash/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/storage.py b/ix-dev/community/filestash/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/filestash/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/validations.py b/ix-dev/community/filestash/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/filestash/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/configs.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_1_0/container.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/filestash/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/depends.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_1_0/deps.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/filestash/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/filestash/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/filestash/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/filestash/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/filestash/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/filestash/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/filestash/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/filestash/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/filestash/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/device.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/device.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_1_0/devices.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/filestash/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/dns.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_1_0/environment.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/filestash/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/error.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/error.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_1_0/functions.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/filestash/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/filestash/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/filestash/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/labels.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/notes.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/portal.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/portals.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/ports.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/render.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/render.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/resources.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/restart.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/storage.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_1_0/validations.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/filestash/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/filestash/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/filestash/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/filestash/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/filestash/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/firefly-iii/app.yaml b/ix-dev/community/firefly-iii/app.yaml index 52c506ee35..1c67e97e91 100644 --- a/ix-dev/community/firefly-iii/app.yaml +++ b/ix-dev/community/firefly-iii/app.yaml @@ -19,8 +19,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/firefly-iii/icons/icon.png keywords: - finance -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -57,4 +57,4 @@ sources: - https://github.com/firefly-iii/firefly-iii title: Firefly III train: community -version: 1.3.6 +version: 1.3.7 diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/container.py b/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/deps.py b/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/devices.py b/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/environment.py b/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/functions.py b/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/storage.py b/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/validations.py b/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/configs.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/container.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/depends.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/deps.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/device.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/device.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/devices.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/dns.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/environment.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/error.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/error.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/functions.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/labels.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/notes.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/portal.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/portals.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/ports.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/render.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/render.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/resources.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/restart.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/storage.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/storage.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_volumes.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_volumes.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/validations.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/firefly-iii/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/firefly-iii/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/firefly-iii/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/firefly-iii/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/flame/app.yaml b/ix-dev/community/flame/app.yaml index 94652289e5..95cb5f4402 100644 --- a/ix-dev/community/flame/app.yaml +++ b/ix-dev/community/flame/app.yaml @@ -14,8 +14,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/flame/icons/icon.png keywords: - startpage -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -34,4 +34,4 @@ sources: - https://github.com/pawelmalak/flame title: Flame train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/container.py b/ix-dev/community/flame/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/flame/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/deps.py b/ix-dev/community/flame/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/flame/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/devices.py b/ix-dev/community/flame/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/flame/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/environment.py b/ix-dev/community/flame/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/flame/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/functions.py b/ix-dev/community/flame/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/flame/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/flame/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/flame/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/storage.py b/ix-dev/community/flame/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/flame/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/validations.py b/ix-dev/community/flame/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/flame/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/flame/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/configs.py b/ix-dev/community/flame/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/flame/templates/library/base_v2_1_0/container.py b/ix-dev/community/flame/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/flame/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/depends.py b/ix-dev/community/flame/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/flame/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/flame/templates/library/base_v2_1_0/deps.py b/ix-dev/community/flame/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/flame/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/flame/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/flame/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/flame/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/flame/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/flame/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/flame/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/flame/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/flame/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/flame/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/flame/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/flame/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/flame/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/device.py b/ix-dev/community/flame/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/device.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/flame/templates/library/base_v2_1_0/devices.py b/ix-dev/community/flame/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/flame/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/dns.py b/ix-dev/community/flame/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/flame/templates/library/base_v2_1_0/environment.py b/ix-dev/community/flame/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/flame/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/error.py b/ix-dev/community/flame/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/error.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/flame/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/flame/templates/library/base_v2_1_0/functions.py b/ix-dev/community/flame/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/flame/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/flame/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/flame/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/flame/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/labels.py b/ix-dev/community/flame/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/notes.py b/ix-dev/community/flame/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/portal.py b/ix-dev/community/flame/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/portals.py b/ix-dev/community/flame/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/ports.py b/ix-dev/community/flame/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/render.py b/ix-dev/community/flame/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/render.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/resources.py b/ix-dev/community/flame/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/restart.py b/ix-dev/community/flame/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/storage.py b/ix-dev/community/flame/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/flame/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/flame/templates/library/base_v2_1_0/validations.py b/ix-dev/community/flame/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/flame/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/flame/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/flame/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/flame/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/flame/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/flame/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/flame/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/flame/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/flame/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/flaresolverr/app.yaml b/ix-dev/community/flaresolverr/app.yaml index ee9a29df39..9a0f98b04e 100644 --- a/ix-dev/community/flaresolverr/app.yaml +++ b/ix-dev/community/flaresolverr/app.yaml @@ -9,8 +9,8 @@ icon: https://media.sys.truenas.net/apps/flaresolverr/icons/icon.svg keywords: - networking - captcha -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -27,4 +27,4 @@ sources: - https://github.com/FlareSolverr/FlareSolverr title: FlareSolverr train: community -version: 1.0.9 +version: 1.0.10 diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/container.py b/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/deps.py b/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/devices.py b/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/environment.py b/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/functions.py b/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/storage.py b/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/validations.py b/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/configs.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/container.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/depends.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/deps.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/device.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/device.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/devices.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/dns.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/environment.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/error.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/error.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/functions.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/labels.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/notes.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/portal.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/portals.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/ports.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/render.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/render.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/resources.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/restart.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/storage.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/sysctls.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/sysctls.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_sysctls.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_sysctls.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/validations.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/flaresolverr/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/flaresolverr/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/flaresolverr/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/flaresolverr/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/freshrss/app.yaml b/ix-dev/community/freshrss/app.yaml index 9f81c389c8..dd3808c5ba 100644 --- a/ix-dev/community/freshrss/app.yaml +++ b/ix-dev/community/freshrss/app.yaml @@ -15,8 +15,8 @@ icon: https://media.sys.truenas.net/apps/freshrss/icons/icon.png keywords: - rss - news -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -41,4 +41,4 @@ sources: - https://hub.docker.com/r/freshrss/freshrss title: FreshRSS train: community -version: 1.2.4 +version: 1.2.5 diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/container.py b/ix-dev/community/freshrss/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/freshrss/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/deps.py b/ix-dev/community/freshrss/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/freshrss/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/devices.py b/ix-dev/community/freshrss/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/freshrss/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/environment.py b/ix-dev/community/freshrss/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/freshrss/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/functions.py b/ix-dev/community/freshrss/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/freshrss/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/freshrss/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/freshrss/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/storage.py b/ix-dev/community/freshrss/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/freshrss/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/validations.py b/ix-dev/community/freshrss/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/freshrss/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/configs.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_1_0/container.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/freshrss/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/depends.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_1_0/deps.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/freshrss/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/freshrss/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/freshrss/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/freshrss/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/freshrss/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/freshrss/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/freshrss/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/freshrss/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/freshrss/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/device.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/device.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_1_0/devices.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/freshrss/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/dns.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_1_0/environment.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/freshrss/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/error.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/error.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_1_0/functions.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/freshrss/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/freshrss/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/freshrss/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/labels.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/notes.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/portal.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/portals.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/ports.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/render.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/render.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/resources.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/restart.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/storage.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_1_0/validations.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/freshrss/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/freshrss/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/freshrss/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/freshrss/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/freshrss/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/frigate/app.yaml b/ix-dev/community/frigate/app.yaml index 1298393bde..739dae350c 100644 --- a/ix-dev/community/frigate/app.yaml +++ b/ix-dev/community/frigate/app.yaml @@ -21,8 +21,8 @@ icon: https://media.sys.truenas.net/apps/frigate/icons/icon.svg keywords: - camera - nvr -lib_version: 2.0.30 -lib_version_hash: a87750c752394f7bf0eb461678564a9ef99bc4e4541787ef25e7be1095a6f879 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -42,4 +42,4 @@ sources: - https://github.com/blakeblackshear/frigate title: Frigate train: community -version: 1.1.7 +version: 1.1.8 diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/container.py b/ix-dev/community/frigate/templates/library/base_v2_0_30/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/frigate/templates/library/base_v2_0_30/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/deps.py b/ix-dev/community/frigate/templates/library/base_v2_0_30/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/frigate/templates/library/base_v2_0_30/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/devices.py b/ix-dev/community/frigate/templates/library/base_v2_0_30/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/frigate/templates/library/base_v2_0_30/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/environment.py b/ix-dev/community/frigate/templates/library/base_v2_0_30/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/frigate/templates/library/base_v2_0_30/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/functions.py b/ix-dev/community/frigate/templates/library/base_v2_0_30/functions.py deleted file mode 100644 index b0c834ab4b..0000000000 --- a/ix-dev/community/frigate/templates/library/base_v2_0_30/functions.py +++ /dev/null @@ -1,142 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _or_default(self, value, default): - if not value: - return default - return value - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - "or_default": self._or_default, - } diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/healthcheck.py b/ix-dev/community/frigate/templates/library/base_v2_0_30/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/frigate/templates/library/base_v2_0_30/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_container.py b/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_deps.py b/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_device.py b/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_environment.py b/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_functions.py b/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_functions.py deleted file mode 100644 index 13d5e49522..0000000000 --- a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_functions.py +++ /dev/null @@ -1,82 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - {"func": "or_default", "values": [None, 1], "expected": 1}, - {"func": "or_default", "values": [1, None], "expected": 1}, - {"func": "or_default", "values": [False, 1], "expected": 1}, - {"func": "or_default", "values": [True, 1], "expected": True}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_healthcheck.py b/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/validations.py b/ix-dev/community/frigate/templates/library/base_v2_0_30/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/frigate/templates/library/base_v2_0_30/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/__init__.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/__init__.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/configs.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/configs.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_1_0/container.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/frigate/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/depends.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/depends.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/deploy.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/deploy.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_1_0/deps.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/frigate/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/frigate/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/frigate/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/frigate/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/frigate/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/frigate/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/frigate/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/frigate/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/frigate/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/device.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/device.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_1_0/devices.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/frigate/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/dns.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/dns.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_1_0/environment.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/frigate/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/error.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/error.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/formatter.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/formatter.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_1_0/functions.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/frigate/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/frigate/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/frigate/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/labels.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/labels.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/notes.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/notes.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/portal.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/portal.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/portals.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/portals.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/ports.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/ports.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/render.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/render.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/resources.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/resources.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/restart.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/restart.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/storage.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/__init__.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/tests/__init__.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_build_image.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_build_image.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_configs.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_configs.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_depends.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_depends.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_dns.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_dns.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_formatter.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_formatter.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_labels.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_labels.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_notes.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_notes.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_portal.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_portal.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_ports.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_ports.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_render.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_render.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_resources.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_resources.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_restart.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/tests/test_restart.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_1_0/validations.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/frigate/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/volume_mount.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/volume_mount.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/volume_mount_types.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/volume_mount_types.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/volume_sources.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/volume_sources.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/volume_types.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/volume_types.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/frigate/templates/library/base_v2_0_30/volumes.py b/ix-dev/community/frigate/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/frigate/templates/library/base_v2_0_30/volumes.py rename to ix-dev/community/frigate/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/fscrawler/app.yaml b/ix-dev/community/fscrawler/app.yaml index 59f38658d2..b69ee9ac5e 100644 --- a/ix-dev/community/fscrawler/app.yaml +++ b/ix-dev/community/fscrawler/app.yaml @@ -10,8 +10,8 @@ icon: https://media.sys.truenas.net/apps/fscrawler/icons/icon.svg keywords: - index - crawler -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -30,4 +30,4 @@ sources: - https://fscrawler.readthedocs.io/ title: FSCrawler train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/container.py b/ix-dev/community/fscrawler/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/deps.py b/ix-dev/community/fscrawler/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/devices.py b/ix-dev/community/fscrawler/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/environment.py b/ix-dev/community/fscrawler/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/functions.py b/ix-dev/community/fscrawler/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/fscrawler/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/storage.py b/ix-dev/community/fscrawler/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/validations.py b/ix-dev/community/fscrawler/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/configs.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_1_0/container.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/depends.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_1_0/deps.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/device.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/device.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_1_0/devices.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/dns.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_1_0/environment.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/error.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/error.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_1_0/functions.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/labels.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/notes.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/portal.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/portals.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/ports.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/render.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/render.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/resources.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/restart.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/storage.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_1_0/validations.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/fscrawler/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/fscrawler/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/fscrawler/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/fscrawler/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/gaseous-server/app.yaml b/ix-dev/community/gaseous-server/app.yaml index 06340a6e6d..ffdde97d73 100644 --- a/ix-dev/community/gaseous-server/app.yaml +++ b/ix-dev/community/gaseous-server/app.yaml @@ -16,8 +16,8 @@ icon: https://media.sys.truenas.net/apps/gaseous-server/icons/icon.png keywords: - games - emulation -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -41,4 +41,4 @@ sources: - https://github.com/gaseous-project/gaseous-server title: Gaseous Server train: community -version: 1.0.5 +version: 1.0.6 diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/container.py b/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/deps.py b/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/devices.py b/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/environment.py b/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/functions.py b/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/storage.py b/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/validations.py b/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/configs.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/container.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/depends.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/deps.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/device.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/device.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/devices.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/dns.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/environment.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/error.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/error.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/functions.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/labels.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/notes.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/portal.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/portals.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/ports.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/render.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/render.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/resources.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/restart.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/storage.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/storage.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_volumes.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_volumes.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/validations.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/gaseous-server/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/gaseous-server/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/gaseous-server/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/gaseous-server/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/gitea/app.yaml b/ix-dev/community/gitea/app.yaml index 70478c04f5..681d13e2d8 100644 --- a/ix-dev/community/gitea/app.yaml +++ b/ix-dev/community/gitea/app.yaml @@ -10,8 +10,8 @@ keywords: - git - gitea - source control -lib_version: 2.0.30 -lib_version_hash: a87750c752394f7bf0eb461678564a9ef99bc4e4541787ef25e7be1095a6f879 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -38,4 +38,4 @@ sources: - https://docs.gitea.io/en-us/install-with-docker-rootless title: Gitea train: community -version: 1.1.4 +version: 1.1.5 diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/container.py b/ix-dev/community/gitea/templates/library/base_v2_0_30/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/gitea/templates/library/base_v2_0_30/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/deps.py b/ix-dev/community/gitea/templates/library/base_v2_0_30/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/gitea/templates/library/base_v2_0_30/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/devices.py b/ix-dev/community/gitea/templates/library/base_v2_0_30/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/gitea/templates/library/base_v2_0_30/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/environment.py b/ix-dev/community/gitea/templates/library/base_v2_0_30/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/gitea/templates/library/base_v2_0_30/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/functions.py b/ix-dev/community/gitea/templates/library/base_v2_0_30/functions.py deleted file mode 100644 index b0c834ab4b..0000000000 --- a/ix-dev/community/gitea/templates/library/base_v2_0_30/functions.py +++ /dev/null @@ -1,142 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _or_default(self, value, default): - if not value: - return default - return value - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - "or_default": self._or_default, - } diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/healthcheck.py b/ix-dev/community/gitea/templates/library/base_v2_0_30/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/gitea/templates/library/base_v2_0_30/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_container.py b/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_deps.py b/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_device.py b/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_environment.py b/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_functions.py b/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_functions.py deleted file mode 100644 index 13d5e49522..0000000000 --- a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_functions.py +++ /dev/null @@ -1,82 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - {"func": "or_default", "values": [None, 1], "expected": 1}, - {"func": "or_default", "values": [1, None], "expected": 1}, - {"func": "or_default", "values": [False, 1], "expected": 1}, - {"func": "or_default", "values": [True, 1], "expected": True}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_healthcheck.py b/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/validations.py b/ix-dev/community/gitea/templates/library/base_v2_0_30/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/gitea/templates/library/base_v2_0_30/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/__init__.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/__init__.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/configs.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/configs.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_1_0/container.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/gitea/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/depends.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/depends.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/deploy.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/deploy.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_1_0/deps.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/gitea/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/gitea/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/gitea/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/gitea/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/gitea/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/gitea/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/gitea/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/gitea/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/gitea/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/device.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/device.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_1_0/devices.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/gitea/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/dns.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/dns.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_1_0/environment.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/gitea/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/error.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/error.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/formatter.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/formatter.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_1_0/functions.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/gitea/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/gitea/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/gitea/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/labels.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/labels.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/notes.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/notes.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/portal.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/portal.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/portals.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/portals.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/ports.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/ports.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/render.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/render.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/resources.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/resources.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/restart.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/restart.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/storage.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/__init__.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/tests/__init__.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_build_image.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_build_image.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_configs.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_configs.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_depends.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_depends.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_dns.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_dns.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_formatter.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_formatter.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_labels.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_labels.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_notes.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_notes.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_portal.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_portal.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_ports.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_ports.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_render.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_render.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_resources.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_resources.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_restart.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/tests/test_restart.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_1_0/validations.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/gitea/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/volume_mount.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/volume_mount.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/volume_mount_types.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/volume_mount_types.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/volume_sources.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/volume_sources.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/volume_types.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/volume_types.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/gitea/templates/library/base_v2_0_30/volumes.py b/ix-dev/community/gitea/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/gitea/templates/library/base_v2_0_30/volumes.py rename to ix-dev/community/gitea/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/grafana/app.yaml b/ix-dev/community/grafana/app.yaml index 0bef0d3026..8d2abf0e7c 100644 --- a/ix-dev/community/grafana/app.yaml +++ b/ix-dev/community/grafana/app.yaml @@ -12,8 +12,8 @@ keywords: - monitoring - metrics - dashboards -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -34,4 +34,4 @@ sources: - https://github.com/grafana title: Grafana train: community -version: 1.2.2 +version: 1.2.3 diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/container.py b/ix-dev/community/grafana/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/grafana/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/deps.py b/ix-dev/community/grafana/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/grafana/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/devices.py b/ix-dev/community/grafana/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/grafana/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/environment.py b/ix-dev/community/grafana/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/grafana/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/functions.py b/ix-dev/community/grafana/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/grafana/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/grafana/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/grafana/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/storage.py b/ix-dev/community/grafana/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/grafana/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/validations.py b/ix-dev/community/grafana/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/grafana/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/configs.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_1_0/container.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/grafana/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/depends.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_1_0/deps.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/grafana/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/grafana/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/grafana/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/grafana/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/grafana/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/grafana/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/grafana/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/grafana/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/grafana/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/device.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/device.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_1_0/devices.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/grafana/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/dns.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_1_0/environment.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/grafana/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/error.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/error.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_1_0/functions.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/grafana/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/grafana/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/grafana/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/labels.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/notes.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/portal.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/portals.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/ports.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/render.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/render.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/resources.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/restart.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/storage.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_1_0/validations.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/grafana/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/grafana/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/grafana/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/grafana/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/grafana/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/handbrake/app.yaml b/ix-dev/community/handbrake/app.yaml index b60cb35afd..8a1edeba42 100644 --- a/ix-dev/community/handbrake/app.yaml +++ b/ix-dev/community/handbrake/app.yaml @@ -28,8 +28,8 @@ keywords: - media - video - transcoder -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -47,4 +47,4 @@ sources: - https://hub.docker.com/r/jlesage/handbrake title: Handbrake train: community -version: 2.1.0 +version: 2.1.1 diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/container.py b/ix-dev/community/handbrake/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/handbrake/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/deps.py b/ix-dev/community/handbrake/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/handbrake/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/devices.py b/ix-dev/community/handbrake/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/handbrake/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/environment.py b/ix-dev/community/handbrake/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/handbrake/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/functions.py b/ix-dev/community/handbrake/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/handbrake/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/handbrake/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/handbrake/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/storage.py b/ix-dev/community/handbrake/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/handbrake/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/validations.py b/ix-dev/community/handbrake/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/handbrake/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/configs.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_1_0/container.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/handbrake/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/depends.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_1_0/deps.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/handbrake/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/handbrake/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/handbrake/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/handbrake/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/handbrake/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/handbrake/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/handbrake/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/handbrake/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/handbrake/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/device.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/device.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_1_0/devices.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/handbrake/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/dns.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_1_0/environment.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/handbrake/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/error.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/error.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_1_0/functions.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/handbrake/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/handbrake/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/handbrake/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/labels.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/notes.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/portal.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/portals.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/ports.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/render.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/render.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/resources.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/restart.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/storage.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_1_0/validations.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/handbrake/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/handbrake/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/handbrake/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/handbrake/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/handbrake/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/homarr/app.yaml b/ix-dev/community/homarr/app.yaml index 2a92d070af..ac8700241b 100644 --- a/ix-dev/community/homarr/app.yaml +++ b/ix-dev/community/homarr/app.yaml @@ -9,8 +9,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/homarr/icons/icon.svg keywords: - dashboard -lib_version: 2.0.23 -lib_version_hash: 8909de0f230ddb07f219f65eb8e67f9b8a112e9c310bee801834248854aa91af +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -31,4 +31,4 @@ sources: - https://github.com/ajnart/homarr title: Homarr train: community -version: 1.1.1 +version: 1.1.2 diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/container.py b/ix-dev/community/homarr/templates/library/base_v2_0_23/container.py deleted file mode 100644 index 34df9ec83b..0000000000 --- a/ix-dev/community/homarr/templates/library/base_v2_0_23/container.py +++ /dev/null @@ -1,304 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/deps.py b/ix-dev/community/homarr/templates/library/base_v2_0_23/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/homarr/templates/library/base_v2_0_23/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/devices.py b/ix-dev/community/homarr/templates/library/base_v2_0_23/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/homarr/templates/library/base_v2_0_23/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/environment.py b/ix-dev/community/homarr/templates/library/base_v2_0_23/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/homarr/templates/library/base_v2_0_23/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/functions.py b/ix-dev/community/homarr/templates/library/base_v2_0_23/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/homarr/templates/library/base_v2_0_23/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/healthcheck.py b/ix-dev/community/homarr/templates/library/base_v2_0_23/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/homarr/templates/library/base_v2_0_23/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/storage.py b/ix-dev/community/homarr/templates/library/base_v2_0_23/storage.py deleted file mode 100644 index bf39d45daa..0000000000 --- a/ix-dev/community/homarr/templates/library/base_v2_0_23/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def _add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_container.py b/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_container.py deleted file mode 100644 index 5e51b86198..0000000000 --- a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_container.py +++ /dev/null @@ -1,294 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_deps.py b/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_device.py b/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_environment.py b/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_functions.py b/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_healthcheck.py b/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_volumes.py b/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_volumes.py deleted file mode 100644 index 2084de09e0..0000000000 --- a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage._add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage._add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage._add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/validations.py b/ix-dev/community/homarr/templates/library/base_v2_0_23/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/homarr/templates/library/base_v2_0_23/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/__init__.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/__init__.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/configs.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/configs.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_1_0/container.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/homarr/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/depends.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/depends.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/deploy.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/deploy.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_1_0/deps.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/homarr/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/homarr/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/homarr/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/homarr/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/homarr/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/homarr/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/homarr/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/homarr/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/homarr/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/device.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/device.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_1_0/devices.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/homarr/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/dns.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/dns.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_1_0/environment.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/homarr/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/error.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/error.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/formatter.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/formatter.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_1_0/functions.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/homarr/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/homarr/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/homarr/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/labels.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/labels.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/notes.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/notes.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/portal.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/portal.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/portals.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/portals.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/ports.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/ports.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/render.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/render.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/resources.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/resources.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/restart.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/restart.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/storage.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/__init__.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/tests/__init__.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_build_image.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_build_image.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_configs.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_configs.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_depends.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_depends.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_dns.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_dns.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_formatter.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_formatter.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_labels.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_labels.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_notes.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_notes.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_portal.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_portal.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_ports.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_ports.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_render.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_render.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_resources.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_resources.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_restart.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/tests/test_restart.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_1_0/validations.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/homarr/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/volume_mount.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/volume_mount.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/volume_mount_types.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/volume_mount_types.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/volume_sources.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/volume_sources.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/volume_types.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/volume_types.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/homarr/templates/library/base_v2_0_23/volumes.py b/ix-dev/community/homarr/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/homarr/templates/library/base_v2_0_23/volumes.py rename to ix-dev/community/homarr/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/homepage/app.yaml b/ix-dev/community/homepage/app.yaml index 3ba783ce6f..01c0bc8540 100644 --- a/ix-dev/community/homepage/app.yaml +++ b/ix-dev/community/homepage/app.yaml @@ -8,8 +8,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/homepage/icons/icon.png keywords: - dashboard -lib_version: 2.0.23 -lib_version_hash: 8909de0f230ddb07f219f65eb8e67f9b8a112e9c310bee801834248854aa91af +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -33,4 +33,4 @@ sources: - https://github.com/benphelps/homepage title: Homepage train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/container.py b/ix-dev/community/homepage/templates/library/base_v2_0_23/container.py deleted file mode 100644 index 34df9ec83b..0000000000 --- a/ix-dev/community/homepage/templates/library/base_v2_0_23/container.py +++ /dev/null @@ -1,304 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/deps.py b/ix-dev/community/homepage/templates/library/base_v2_0_23/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/homepage/templates/library/base_v2_0_23/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/devices.py b/ix-dev/community/homepage/templates/library/base_v2_0_23/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/homepage/templates/library/base_v2_0_23/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/environment.py b/ix-dev/community/homepage/templates/library/base_v2_0_23/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/homepage/templates/library/base_v2_0_23/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/functions.py b/ix-dev/community/homepage/templates/library/base_v2_0_23/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/homepage/templates/library/base_v2_0_23/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/healthcheck.py b/ix-dev/community/homepage/templates/library/base_v2_0_23/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/homepage/templates/library/base_v2_0_23/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/storage.py b/ix-dev/community/homepage/templates/library/base_v2_0_23/storage.py deleted file mode 100644 index bf39d45daa..0000000000 --- a/ix-dev/community/homepage/templates/library/base_v2_0_23/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def _add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_container.py b/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_container.py deleted file mode 100644 index 5e51b86198..0000000000 --- a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_container.py +++ /dev/null @@ -1,294 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_deps.py b/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_device.py b/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_environment.py b/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_functions.py b/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_healthcheck.py b/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_volumes.py b/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_volumes.py deleted file mode 100644 index 2084de09e0..0000000000 --- a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage._add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage._add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage._add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/validations.py b/ix-dev/community/homepage/templates/library/base_v2_0_23/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/homepage/templates/library/base_v2_0_23/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/__init__.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/__init__.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/configs.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/configs.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_1_0/container.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/homepage/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/depends.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/depends.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/deploy.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/deploy.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_1_0/deps.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/homepage/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/homepage/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/homepage/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/homepage/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/homepage/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/homepage/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/homepage/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/homepage/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/homepage/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/device.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/device.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_1_0/devices.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/homepage/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/dns.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/dns.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_1_0/environment.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/homepage/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/error.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/error.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/formatter.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/formatter.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_1_0/functions.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/homepage/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/homepage/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/homepage/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/labels.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/labels.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/notes.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/notes.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/portal.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/portal.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/portals.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/portals.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/ports.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/ports.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/render.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/render.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/resources.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/resources.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/restart.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/restart.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/storage.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/sysctls.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/sysctls.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/__init__.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/tests/__init__.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_build_image.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_build_image.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_configs.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_configs.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_depends.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_depends.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_dns.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_dns.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_formatter.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_formatter.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_labels.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_labels.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_notes.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_notes.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_portal.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_portal.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_ports.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_ports.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_render.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_render.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_resources.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_resources.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_restart.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/tests/test_restart.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_sysctls.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_sysctls.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_1_0/validations.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/homepage/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/volume_mount.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/volume_mount.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/volume_mount_types.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/volume_mount_types.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/volume_sources.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/volume_sources.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/volume_types.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/volume_types.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/homepage/templates/library/base_v2_0_23/volumes.py b/ix-dev/community/homepage/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/homepage/templates/library/base_v2_0_23/volumes.py rename to ix-dev/community/homepage/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/homer/app.yaml b/ix-dev/community/homer/app.yaml index d8b07aeb97..d4e4723d9f 100644 --- a/ix-dev/community/homer/app.yaml +++ b/ix-dev/community/homer/app.yaml @@ -7,8 +7,8 @@ description: Homer is a dead simple static HOMepage for your servER to keep your home: https://github.com/bastienwirtz/homer host_mounts: [] icon: https://media.sys.truenas.net/apps/homer/icons/icon.png -lib_version: 2.0.24 -lib_version_hash: 283cc9c5d0a45474968e1280324fab8fc7e176c41fdafdb6dcf9b9a74efebb9c +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -30,4 +30,4 @@ tags: - homepage title: Homer train: community -version: 2.1.2 +version: 2.1.3 diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/container.py b/ix-dev/community/homer/templates/library/base_v2_0_24/container.py deleted file mode 100644 index e49370c28b..0000000000 --- a/ix-dev/community/homer/templates/library/base_v2_0_24/container.py +++ /dev/null @@ -1,307 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/deps.py b/ix-dev/community/homer/templates/library/base_v2_0_24/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/homer/templates/library/base_v2_0_24/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/devices.py b/ix-dev/community/homer/templates/library/base_v2_0_24/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/homer/templates/library/base_v2_0_24/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/environment.py b/ix-dev/community/homer/templates/library/base_v2_0_24/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/homer/templates/library/base_v2_0_24/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/functions.py b/ix-dev/community/homer/templates/library/base_v2_0_24/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/homer/templates/library/base_v2_0_24/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/healthcheck.py b/ix-dev/community/homer/templates/library/base_v2_0_24/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/homer/templates/library/base_v2_0_24/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_container.py b/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_container.py deleted file mode 100644 index 84d712a403..0000000000 --- a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_container.py +++ /dev/null @@ -1,314 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_deps.py b/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_device.py b/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_environment.py b/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_functions.py b/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_healthcheck.py b/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/validations.py b/ix-dev/community/homer/templates/library/base_v2_0_24/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/homer/templates/library/base_v2_0_24/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/__init__.py b/ix-dev/community/homer/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/__init__.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/configs.py b/ix-dev/community/homer/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/configs.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/homer/templates/library/base_v2_1_0/container.py b/ix-dev/community/homer/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/homer/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/depends.py b/ix-dev/community/homer/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/depends.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/deploy.py b/ix-dev/community/homer/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/deploy.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/homer/templates/library/base_v2_1_0/deps.py b/ix-dev/community/homer/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/homer/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/homer/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/homer/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/homer/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/homer/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/homer/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/homer/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/homer/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/homer/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/homer/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/homer/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/homer/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/homer/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/device.py b/ix-dev/community/homer/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/device.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/homer/templates/library/base_v2_1_0/devices.py b/ix-dev/community/homer/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/homer/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/dns.py b/ix-dev/community/homer/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/dns.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/homer/templates/library/base_v2_1_0/environment.py b/ix-dev/community/homer/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/homer/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/error.py b/ix-dev/community/homer/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/error.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/formatter.py b/ix-dev/community/homer/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/formatter.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/homer/templates/library/base_v2_1_0/functions.py b/ix-dev/community/homer/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/homer/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/homer/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/homer/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/homer/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/labels.py b/ix-dev/community/homer/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/labels.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/notes.py b/ix-dev/community/homer/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/notes.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/portal.py b/ix-dev/community/homer/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/portal.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/portals.py b/ix-dev/community/homer/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/portals.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/ports.py b/ix-dev/community/homer/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/ports.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/render.py b/ix-dev/community/homer/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/render.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/resources.py b/ix-dev/community/homer/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/resources.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/restart.py b/ix-dev/community/homer/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/restart.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/storage.py b/ix-dev/community/homer/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/homer/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/__init__.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/tests/__init__.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_build_image.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_build_image.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_configs.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_configs.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_depends.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_depends.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_dns.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_dns.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_formatter.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_formatter.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_labels.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_labels.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_notes.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_notes.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_portal.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_portal.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_ports.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_ports.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_render.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_render.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_resources.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_resources.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_restart.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/tests/test_restart.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/homer/templates/library/base_v2_1_0/validations.py b/ix-dev/community/homer/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/homer/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/volume_mount.py b/ix-dev/community/homer/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/volume_mount.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/volume_mount_types.py b/ix-dev/community/homer/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/volume_mount_types.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/volume_sources.py b/ix-dev/community/homer/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/volume_sources.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/volume_types.py b/ix-dev/community/homer/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/volume_types.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/homer/templates/library/base_v2_0_24/volumes.py b/ix-dev/community/homer/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/homer/templates/library/base_v2_0_24/volumes.py rename to ix-dev/community/homer/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/iconik-storage-gateway/app.yaml b/ix-dev/community/iconik-storage-gateway/app.yaml index 4c6e4bda8b..d993c98931 100644 --- a/ix-dev/community/iconik-storage-gateway/app.yaml +++ b/ix-dev/community/iconik-storage-gateway/app.yaml @@ -9,8 +9,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/iconik-storage-gateway/icons/icon.svg keywords: - iconik -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -28,4 +28,4 @@ sources: - https://app.iconik.io/help/pages/isg title: Iconik Storage Gateway train: community -version: 1.0.1 +version: 1.0.2 diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/container.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/deps.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/devices.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/environment.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/functions.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/storage.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/validations.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/configs.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/container.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/depends.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/deps.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/device.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/device.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/devices.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/dns.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/environment.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/error.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/error.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/functions.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/labels.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/notes.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/portal.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/portals.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/ports.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/render.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/render.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/resources.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/restart.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/storage.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/storage.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_volumes.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_volumes.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/validations.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/iconik-storage-gateway/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/iconik-storage-gateway/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/immich/app.yaml b/ix-dev/community/immich/app.yaml index 92c71739bf..bd0b9a8086 100644 --- a/ix-dev/community/immich/app.yaml +++ b/ix-dev/community/immich/app.yaml @@ -16,8 +16,8 @@ icon: https://media.sys.truenas.net/apps/immich/icons/icon.svg keywords: - photos - backup -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -45,4 +45,4 @@ sources: - https://github.com/immich-app/immich title: Immich train: community -version: 1.7.8 +version: 1.7.9 diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/container.py b/ix-dev/community/immich/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/immich/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/deps.py b/ix-dev/community/immich/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/immich/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/devices.py b/ix-dev/community/immich/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/immich/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/environment.py b/ix-dev/community/immich/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/immich/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/functions.py b/ix-dev/community/immich/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/immich/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/immich/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/immich/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/storage.py b/ix-dev/community/immich/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/immich/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/validations.py b/ix-dev/community/immich/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/immich/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/immich/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/configs.py b/ix-dev/community/immich/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/immich/templates/library/base_v2_1_0/container.py b/ix-dev/community/immich/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/immich/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/depends.py b/ix-dev/community/immich/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/immich/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/immich/templates/library/base_v2_1_0/deps.py b/ix-dev/community/immich/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/immich/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/immich/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/immich/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/immich/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/immich/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/immich/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/immich/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/immich/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/immich/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/immich/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/immich/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/immich/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/immich/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/device.py b/ix-dev/community/immich/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/device.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/immich/templates/library/base_v2_1_0/devices.py b/ix-dev/community/immich/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/immich/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/dns.py b/ix-dev/community/immich/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/immich/templates/library/base_v2_1_0/environment.py b/ix-dev/community/immich/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/immich/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/error.py b/ix-dev/community/immich/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/error.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/immich/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/immich/templates/library/base_v2_1_0/functions.py b/ix-dev/community/immich/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/immich/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/immich/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/immich/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/immich/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/labels.py b/ix-dev/community/immich/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/notes.py b/ix-dev/community/immich/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/portal.py b/ix-dev/community/immich/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/portals.py b/ix-dev/community/immich/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/ports.py b/ix-dev/community/immich/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/render.py b/ix-dev/community/immich/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/render.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/resources.py b/ix-dev/community/immich/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/restart.py b/ix-dev/community/immich/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/storage.py b/ix-dev/community/immich/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/immich/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/immich/templates/library/base_v2_1_0/validations.py b/ix-dev/community/immich/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/immich/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/immich/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/immich/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/immich/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/immich/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/immich/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/immich/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/immich/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/immich/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/invidious/app.yaml b/ix-dev/community/invidious/app.yaml index 765a92eda0..6b698e5f65 100644 --- a/ix-dev/community/invidious/app.yaml +++ b/ix-dev/community/invidious/app.yaml @@ -8,8 +8,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/invidious/icons/icon.svg keywords: - youtube -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -38,4 +38,4 @@ sources: - https://quay.io/repository/invidious title: Invidious train: community -version: 1.1.3 +version: 1.1.4 diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/container.py b/ix-dev/community/invidious/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/invidious/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/deps.py b/ix-dev/community/invidious/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/invidious/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/devices.py b/ix-dev/community/invidious/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/invidious/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/environment.py b/ix-dev/community/invidious/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/invidious/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/functions.py b/ix-dev/community/invidious/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/invidious/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/invidious/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/invidious/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/storage.py b/ix-dev/community/invidious/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/invidious/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/validations.py b/ix-dev/community/invidious/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/invidious/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/configs.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_1_0/container.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/invidious/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/depends.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_1_0/deps.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/invidious/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/invidious/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/invidious/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/invidious/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/invidious/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/invidious/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/invidious/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/invidious/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/invidious/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/device.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/device.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_1_0/devices.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/invidious/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/dns.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_1_0/environment.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/invidious/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/error.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/error.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_1_0/functions.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/invidious/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/invidious/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/invidious/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/labels.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/notes.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/portal.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/portals.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/ports.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/render.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/render.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/resources.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/restart.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/storage.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/sysctls.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/sysctls.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_sysctls.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_sysctls.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_1_0/validations.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/invidious/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/invidious/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/invidious/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/invidious/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/invidious/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/ipfs/app.yaml b/ix-dev/community/ipfs/app.yaml index 459e2f081f..bac7b07417 100644 --- a/ix-dev/community/ipfs/app.yaml +++ b/ix-dev/community/ipfs/app.yaml @@ -12,8 +12,8 @@ keywords: - ipfs - file-sharing - kubo -lib_version: 2.0.24 -lib_version_hash: 283cc9c5d0a45474968e1280324fab8fc7e176c41fdafdb6dcf9b9a74efebb9c +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -33,4 +33,4 @@ sources: - https://ipfs.tech/ title: IPFS train: community -version: 1.1.1 +version: 1.1.2 diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/container.py b/ix-dev/community/ipfs/templates/library/base_v2_0_24/container.py deleted file mode 100644 index e49370c28b..0000000000 --- a/ix-dev/community/ipfs/templates/library/base_v2_0_24/container.py +++ /dev/null @@ -1,307 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/deps.py b/ix-dev/community/ipfs/templates/library/base_v2_0_24/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/ipfs/templates/library/base_v2_0_24/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/devices.py b/ix-dev/community/ipfs/templates/library/base_v2_0_24/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/ipfs/templates/library/base_v2_0_24/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/environment.py b/ix-dev/community/ipfs/templates/library/base_v2_0_24/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/ipfs/templates/library/base_v2_0_24/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/functions.py b/ix-dev/community/ipfs/templates/library/base_v2_0_24/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/ipfs/templates/library/base_v2_0_24/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/healthcheck.py b/ix-dev/community/ipfs/templates/library/base_v2_0_24/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/ipfs/templates/library/base_v2_0_24/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_container.py b/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_container.py deleted file mode 100644 index 84d712a403..0000000000 --- a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_container.py +++ /dev/null @@ -1,314 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_deps.py b/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_device.py b/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_environment.py b/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_functions.py b/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_healthcheck.py b/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/validations.py b/ix-dev/community/ipfs/templates/library/base_v2_0_24/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/ipfs/templates/library/base_v2_0_24/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/__init__.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/__init__.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/configs.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/configs.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_1_0/container.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/ipfs/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/depends.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/depends.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/deploy.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/deploy.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_1_0/deps.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/ipfs/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/ipfs/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/ipfs/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/ipfs/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/ipfs/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/ipfs/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/ipfs/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/ipfs/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/ipfs/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/device.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/device.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_1_0/devices.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/ipfs/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/dns.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/dns.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_1_0/environment.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/ipfs/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/error.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/error.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/formatter.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/formatter.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_1_0/functions.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/ipfs/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/ipfs/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/ipfs/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/labels.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/labels.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/notes.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/notes.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/portal.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/portal.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/portals.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/portals.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/ports.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/ports.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/render.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/render.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/resources.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/resources.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/restart.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/restart.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/storage.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/storage.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/sysctls.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/sysctls.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/__init__.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/__init__.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_build_image.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_build_image.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_configs.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_configs.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_depends.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_depends.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_dns.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_dns.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_formatter.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_formatter.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_labels.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_labels.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_notes.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_notes.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_portal.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_portal.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_ports.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_ports.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_render.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_render.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_resources.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_resources.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_restart.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/tests/test_restart.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_sysctls.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_sysctls.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_volumes.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_volumes.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_1_0/validations.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/ipfs/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/volume_mount.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/volume_mount.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/volume_mount_types.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/volume_mount_types.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/volume_sources.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/volume_sources.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/volume_types.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/volume_types.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/ipfs/templates/library/base_v2_0_24/volumes.py b/ix-dev/community/ipfs/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/ipfs/templates/library/base_v2_0_24/volumes.py rename to ix-dev/community/ipfs/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/jellyfin/app.yaml b/ix-dev/community/jellyfin/app.yaml index d5a91d1a28..02c2e37b97 100644 --- a/ix-dev/community/jellyfin/app.yaml +++ b/ix-dev/community/jellyfin/app.yaml @@ -14,8 +14,8 @@ keywords: - tv - media - streaming -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -35,4 +35,4 @@ sources: - https://jellyfin.org/ title: Jellyfin train: community -version: 1.1.7 +version: 1.1.8 diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/container.py b/ix-dev/community/jellyfin/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/deps.py b/ix-dev/community/jellyfin/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/devices.py b/ix-dev/community/jellyfin/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/environment.py b/ix-dev/community/jellyfin/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/functions.py b/ix-dev/community/jellyfin/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/jellyfin/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/storage.py b/ix-dev/community/jellyfin/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/validations.py b/ix-dev/community/jellyfin/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/configs.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_1_0/container.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/depends.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_1_0/deps.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/device.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/device.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_1_0/devices.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/dns.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_1_0/environment.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/error.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/error.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_1_0/functions.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/labels.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/notes.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/portal.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/portals.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/ports.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/render.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/render.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/resources.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/restart.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/storage.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/sysctls.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/sysctls.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_sysctls.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_sysctls.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_1_0/validations.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/jellyfin/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/jellyfin/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/jellyfin/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/jellyfin/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/jellyseerr/app.yaml b/ix-dev/community/jellyseerr/app.yaml index c36064b3a2..f03dcb1202 100644 --- a/ix-dev/community/jellyseerr/app.yaml +++ b/ix-dev/community/jellyseerr/app.yaml @@ -9,8 +9,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/jellyseerr/icons/icon.svg keywords: - media -lib_version: 2.0.24 -lib_version_hash: 283cc9c5d0a45474968e1280324fab8fc7e176c41fdafdb6dcf9b9a74efebb9c +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -29,4 +29,4 @@ sources: - https://hub.docker.com/r/fallenbagel/jellyseerr title: Jellyseerr train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/container.py b/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/container.py deleted file mode 100644 index e49370c28b..0000000000 --- a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/container.py +++ /dev/null @@ -1,307 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/deps.py b/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/devices.py b/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/environment.py b/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/functions.py b/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/healthcheck.py b/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_container.py b/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_container.py deleted file mode 100644 index 84d712a403..0000000000 --- a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_container.py +++ /dev/null @@ -1,314 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_deps.py b/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_device.py b/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_environment.py b/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_functions.py b/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_healthcheck.py b/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/validations.py b/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/__init__.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/__init__.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/configs.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/configs.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/container.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/depends.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/depends.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/deploy.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/deploy.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/deps.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/device.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/device.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/devices.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/dns.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/dns.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/environment.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/error.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/error.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/formatter.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/formatter.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/functions.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/labels.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/labels.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/notes.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/notes.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/portal.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/portal.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/portals.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/portals.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/ports.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/ports.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/render.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/render.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/resources.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/resources.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/restart.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/restart.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/storage.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/sysctls.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/sysctls.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/sysctls.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/sysctls.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/__init__.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/__init__.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_build_image.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_build_image.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_configs.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_configs.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_depends.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_depends.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_dns.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_dns.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_formatter.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_formatter.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_labels.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_labels.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_notes.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_notes.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_portal.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_portal.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_ports.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_ports.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_render.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_render.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_resources.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_resources.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_restart.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/tests/test_restart.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_sysctls.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_sysctls.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_sysctls.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_sysctls.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/validations.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/volume_mount.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/volume_mount.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/volume_mount_types.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/volume_mount_types.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/volume_sources.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/volume_sources.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/volume_types.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/volume_types.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/jellyseerr/templates/library/base_v2_0_24/volumes.py b/ix-dev/community/jellyseerr/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/jellyseerr/templates/library/base_v2_0_24/volumes.py rename to ix-dev/community/jellyseerr/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/jenkins/app.yaml b/ix-dev/community/jenkins/app.yaml index b0ab8bf862..3547fb2bec 100644 --- a/ix-dev/community/jenkins/app.yaml +++ b/ix-dev/community/jenkins/app.yaml @@ -9,8 +9,8 @@ icon: https://media.sys.truenas.net/apps/jenkins/icons/icon.svg keywords: - automation - ci/cd -lib_version: 2.0.24 -lib_version_hash: 283cc9c5d0a45474968e1280324fab8fc7e176c41fdafdb6dcf9b9a74efebb9c +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -31,4 +31,4 @@ sources: - https://www.jenkins.io/ title: Jenkins train: community -version: 1.1.1 +version: 1.1.2 diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/container.py b/ix-dev/community/jenkins/templates/library/base_v2_0_24/container.py deleted file mode 100644 index e49370c28b..0000000000 --- a/ix-dev/community/jenkins/templates/library/base_v2_0_24/container.py +++ /dev/null @@ -1,307 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/deps.py b/ix-dev/community/jenkins/templates/library/base_v2_0_24/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/jenkins/templates/library/base_v2_0_24/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/devices.py b/ix-dev/community/jenkins/templates/library/base_v2_0_24/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/jenkins/templates/library/base_v2_0_24/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/environment.py b/ix-dev/community/jenkins/templates/library/base_v2_0_24/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/jenkins/templates/library/base_v2_0_24/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/functions.py b/ix-dev/community/jenkins/templates/library/base_v2_0_24/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/jenkins/templates/library/base_v2_0_24/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/healthcheck.py b/ix-dev/community/jenkins/templates/library/base_v2_0_24/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/jenkins/templates/library/base_v2_0_24/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_container.py b/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_container.py deleted file mode 100644 index 84d712a403..0000000000 --- a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_container.py +++ /dev/null @@ -1,314 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_deps.py b/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_device.py b/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_environment.py b/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_functions.py b/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_healthcheck.py b/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/validations.py b/ix-dev/community/jenkins/templates/library/base_v2_0_24/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/jenkins/templates/library/base_v2_0_24/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/__init__.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/__init__.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/configs.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/configs.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/container.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/depends.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/depends.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/deploy.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/deploy.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/deps.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/device.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/device.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/devices.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/dns.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/dns.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/environment.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/error.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/error.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/formatter.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/formatter.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/functions.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/labels.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/labels.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/notes.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/notes.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/portal.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/portal.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/portals.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/portals.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/ports.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/ports.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/render.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/render.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/resources.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/resources.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/restart.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/restart.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/storage.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/__init__.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/__init__.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_build_image.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_build_image.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_configs.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_configs.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_depends.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_depends.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_dns.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_dns.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_formatter.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_formatter.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_labels.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_labels.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_notes.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_notes.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_portal.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_portal.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_ports.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_ports.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_render.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_render.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_resources.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_resources.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_restart.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/tests/test_restart.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_1_0/validations.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/jenkins/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/volume_mount.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/volume_mount.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/volume_mount_types.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/volume_mount_types.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/volume_sources.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/volume_sources.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/volume_types.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/volume_types.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/jenkins/templates/library/base_v2_0_24/volumes.py b/ix-dev/community/jenkins/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/jenkins/templates/library/base_v2_0_24/volumes.py rename to ix-dev/community/jenkins/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/joplin/app.yaml b/ix-dev/community/joplin/app.yaml index e07bbe42bb..0d8d8c0fd1 100644 --- a/ix-dev/community/joplin/app.yaml +++ b/ix-dev/community/joplin/app.yaml @@ -9,8 +9,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/joplin/icons/icon.png keywords: - notes -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -36,4 +36,4 @@ sources: - https://hub.docker.com/r/joplin/server/ title: Joplin train: community -version: 1.2.3 +version: 1.2.4 diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/container.py b/ix-dev/community/joplin/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/joplin/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/deps.py b/ix-dev/community/joplin/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/joplin/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/devices.py b/ix-dev/community/joplin/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/joplin/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/environment.py b/ix-dev/community/joplin/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/joplin/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/functions.py b/ix-dev/community/joplin/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/joplin/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/joplin/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/joplin/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/storage.py b/ix-dev/community/joplin/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/joplin/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/validations.py b/ix-dev/community/joplin/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/joplin/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/configs.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/container.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/depends.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/deps.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/device.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/device.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/devices.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/dns.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/environment.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/error.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/error.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/functions.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/labels.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/notes.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/portal.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/portals.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/ports.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/render.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/render.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/resources.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/restart.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/storage.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/storage.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_volumes.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_volumes.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_1_0/validations.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/joplin/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/joplin/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/joplin/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/joplin/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/joplin/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/kapowarr/app.yaml b/ix-dev/community/kapowarr/app.yaml index a6a38b0fed..e4f2a9b06a 100644 --- a/ix-dev/community/kapowarr/app.yaml +++ b/ix-dev/community/kapowarr/app.yaml @@ -10,8 +10,8 @@ icon: https://media.sys.truenas.net/apps/kapowarr/icons/icon.svg keywords: - comic - media -lib_version: 2.0.25 -lib_version_hash: 37aa2a7f78f8ef1e8f3d705f4ce5d8bb66eddd4f023c25294a53a893b21a6048 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -31,4 +31,4 @@ sources: - https://github.com/Casvt/Kapowarr title: Kapowarr train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/container.py b/ix-dev/community/kapowarr/templates/library/base_v2_0_25/container.py deleted file mode 100644 index 440926dd5b..0000000000 --- a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/container.py +++ /dev/null @@ -1,313 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/deps.py b/ix-dev/community/kapowarr/templates/library/base_v2_0_25/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/devices.py b/ix-dev/community/kapowarr/templates/library/base_v2_0_25/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/environment.py b/ix-dev/community/kapowarr/templates/library/base_v2_0_25/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/functions.py b/ix-dev/community/kapowarr/templates/library/base_v2_0_25/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/healthcheck.py b/ix-dev/community/kapowarr/templates/library/base_v2_0_25/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_container.py b/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_container.py deleted file mode 100644 index 84d712a403..0000000000 --- a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_container.py +++ /dev/null @@ -1,314 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_deps.py b/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_device.py b/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_environment.py b/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_functions.py b/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_healthcheck.py b/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/validations.py b/ix-dev/community/kapowarr/templates/library/base_v2_0_25/validations.py deleted file mode 100644 index 155c2e1091..0000000000 --- a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/__init__.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/__init__.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/configs.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/configs.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/container.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/depends.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/depends.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/deploy.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/deploy.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/deps.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/device.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/device.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/devices.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/dns.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/dns.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/environment.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/error.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/error.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/formatter.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/formatter.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/functions.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/labels.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/labels.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/notes.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/notes.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/portal.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/portal.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/portals.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/portals.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/ports.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/ports.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/render.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/render.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/resources.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/resources.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/restart.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/restart.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/storage.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/storage.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/__init__.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/__init__.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_build_image.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_build_image.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_configs.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_configs.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_depends.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_depends.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_dns.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_dns.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_formatter.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_formatter.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_labels.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_labels.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_notes.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_notes.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_portal.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_portal.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_ports.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_ports.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_render.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_render.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_resources.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_resources.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_restart.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/tests/test_restart.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_volumes.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_volumes.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_1_0/validations.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/volume_mount.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/volume_mount.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/volume_mount_types.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/volume_mount_types.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/volume_sources.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/volume_sources.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/volume_types.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/volume_types.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/kapowarr/templates/library/base_v2_0_25/volumes.py b/ix-dev/community/kapowarr/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/kapowarr/templates/library/base_v2_0_25/volumes.py rename to ix-dev/community/kapowarr/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/kavita/app.yaml b/ix-dev/community/kavita/app.yaml index 37de2eaf17..2a646724ae 100644 --- a/ix-dev/community/kavita/app.yaml +++ b/ix-dev/community/kavita/app.yaml @@ -20,8 +20,8 @@ icon: https://media.sys.truenas.net/apps/kavita/icons/icon.png keywords: - ebook - manga -lib_version: 2.0.25 -lib_version_hash: 37aa2a7f78f8ef1e8f3d705f4ce5d8bb66eddd4f023c25294a53a893b21a6048 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -44,4 +44,4 @@ sources: - https://www.kavitareader.com title: Kavita train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/container.py b/ix-dev/community/kavita/templates/library/base_v2_0_25/container.py deleted file mode 100644 index 440926dd5b..0000000000 --- a/ix-dev/community/kavita/templates/library/base_v2_0_25/container.py +++ /dev/null @@ -1,313 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/deps.py b/ix-dev/community/kavita/templates/library/base_v2_0_25/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/kavita/templates/library/base_v2_0_25/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/devices.py b/ix-dev/community/kavita/templates/library/base_v2_0_25/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/kavita/templates/library/base_v2_0_25/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/environment.py b/ix-dev/community/kavita/templates/library/base_v2_0_25/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/kavita/templates/library/base_v2_0_25/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/functions.py b/ix-dev/community/kavita/templates/library/base_v2_0_25/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/kavita/templates/library/base_v2_0_25/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/healthcheck.py b/ix-dev/community/kavita/templates/library/base_v2_0_25/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/kavita/templates/library/base_v2_0_25/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_container.py b/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_container.py deleted file mode 100644 index 84d712a403..0000000000 --- a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_container.py +++ /dev/null @@ -1,314 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_deps.py b/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_device.py b/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_environment.py b/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_functions.py b/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_healthcheck.py b/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/validations.py b/ix-dev/community/kavita/templates/library/base_v2_0_25/validations.py deleted file mode 100644 index 155c2e1091..0000000000 --- a/ix-dev/community/kavita/templates/library/base_v2_0_25/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/__init__.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/__init__.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/configs.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/configs.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/container.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/depends.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/depends.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/deploy.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/deploy.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/deps.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/device.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/device.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/devices.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/dns.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/dns.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/environment.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/error.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/error.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/formatter.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/formatter.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/functions.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/labels.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/labels.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/notes.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/notes.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/portal.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/portal.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/portals.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/portals.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/ports.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/ports.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/render.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/render.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/resources.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/resources.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/restart.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/restart.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/storage.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/storage.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/__init__.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/tests/__init__.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_build_image.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_build_image.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_configs.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_configs.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_depends.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_depends.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_dns.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_dns.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_formatter.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_formatter.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_labels.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_labels.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_notes.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_notes.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_portal.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_portal.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_ports.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_ports.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_render.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_render.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_resources.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_resources.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_restart.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/tests/test_restart.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_volumes.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_volumes.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_1_0/validations.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/kavita/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/volume_mount.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/volume_mount.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/volume_mount_types.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/volume_mount_types.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/volume_sources.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/volume_sources.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/volume_types.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/volume_types.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/kavita/templates/library/base_v2_0_25/volumes.py b/ix-dev/community/kavita/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/kavita/templates/library/base_v2_0_25/volumes.py rename to ix-dev/community/kavita/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/komga/app.yaml b/ix-dev/community/komga/app.yaml index 6883a3d533..0b256f80e3 100644 --- a/ix-dev/community/komga/app.yaml +++ b/ix-dev/community/komga/app.yaml @@ -10,8 +10,8 @@ keywords: - media - comics - mangas -lib_version: 2.0.26 -lib_version_hash: e82ef6b594774d6160be29cae52fd4f8367518498ea1cd386d7d9edc4ea8b654 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -31,4 +31,4 @@ sources: - https://hub.docker.com/r/gotson/komga title: Komga train: community -version: 1.2.0 +version: 1.2.1 diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/container.py b/ix-dev/community/komga/templates/library/base_v2_0_26/container.py deleted file mode 100644 index 440926dd5b..0000000000 --- a/ix-dev/community/komga/templates/library/base_v2_0_26/container.py +++ /dev/null @@ -1,313 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/deps.py b/ix-dev/community/komga/templates/library/base_v2_0_26/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/komga/templates/library/base_v2_0_26/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/devices.py b/ix-dev/community/komga/templates/library/base_v2_0_26/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/komga/templates/library/base_v2_0_26/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/environment.py b/ix-dev/community/komga/templates/library/base_v2_0_26/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/komga/templates/library/base_v2_0_26/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/functions.py b/ix-dev/community/komga/templates/library/base_v2_0_26/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/komga/templates/library/base_v2_0_26/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/healthcheck.py b/ix-dev/community/komga/templates/library/base_v2_0_26/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/komga/templates/library/base_v2_0_26/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_container.py b/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_container.py deleted file mode 100644 index 84d712a403..0000000000 --- a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_container.py +++ /dev/null @@ -1,314 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_deps.py b/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_device.py b/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_environment.py b/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_functions.py b/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_healthcheck.py b/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/validations.py b/ix-dev/community/komga/templates/library/base_v2_0_26/validations.py deleted file mode 100644 index 155c2e1091..0000000000 --- a/ix-dev/community/komga/templates/library/base_v2_0_26/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/__init__.py b/ix-dev/community/komga/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/__init__.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/configs.py b/ix-dev/community/komga/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/configs.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/container.py b/ix-dev/community/komga/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/depends.py b/ix-dev/community/komga/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/depends.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/deploy.py b/ix-dev/community/komga/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/deploy.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/deps.py b/ix-dev/community/komga/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/komga/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/komga/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/komga/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/komga/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/device.py b/ix-dev/community/komga/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/device.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/devices.py b/ix-dev/community/komga/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/dns.py b/ix-dev/community/komga/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/dns.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/environment.py b/ix-dev/community/komga/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/error.py b/ix-dev/community/komga/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/error.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/formatter.py b/ix-dev/community/komga/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/formatter.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/functions.py b/ix-dev/community/komga/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/komga/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/labels.py b/ix-dev/community/komga/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/labels.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/notes.py b/ix-dev/community/komga/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/notes.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/portal.py b/ix-dev/community/komga/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/portal.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/portals.py b/ix-dev/community/komga/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/portals.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/ports.py b/ix-dev/community/komga/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/ports.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/render.py b/ix-dev/community/komga/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/render.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/resources.py b/ix-dev/community/komga/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/resources.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/restart.py b/ix-dev/community/komga/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/restart.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/storage.py b/ix-dev/community/komga/templates/library/base_v2_1_0/storage.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/storage.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/storage.py diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/komga/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/__init__.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/tests/__init__.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_build_image.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_build_image.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_configs.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_configs.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_depends.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_depends.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_dns.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_dns.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_formatter.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_formatter.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_labels.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_labels.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_notes.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_notes.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_portal.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_portal.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_ports.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_ports.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_render.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_render.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_resources.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_resources.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_restart.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/tests/test_restart.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_volumes.py b/ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_volumes.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_volumes.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/tests/test_volumes.py diff --git a/ix-dev/community/komga/templates/library/base_v2_1_0/validations.py b/ix-dev/community/komga/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/komga/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/volume_mount.py b/ix-dev/community/komga/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/volume_mount.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/volume_mount_types.py b/ix-dev/community/komga/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/volume_mount_types.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/volume_sources.py b/ix-dev/community/komga/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/volume_sources.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/volume_types.py b/ix-dev/community/komga/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/volume_types.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/komga/templates/library/base_v2_0_26/volumes.py b/ix-dev/community/komga/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/komga/templates/library/base_v2_0_26/volumes.py rename to ix-dev/community/komga/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/lidarr/app.yaml b/ix-dev/community/lidarr/app.yaml index 125fe2851b..fcb7cd8490 100644 --- a/ix-dev/community/lidarr/app.yaml +++ b/ix-dev/community/lidarr/app.yaml @@ -9,8 +9,8 @@ icon: https://media.sys.truenas.net/apps/lidarr/icons/icon.png keywords: - media - music -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -31,4 +31,4 @@ sources: - https://github.com/Lidarr/Lidarr title: Lidarr train: community -version: 1.2.1 +version: 1.2.2 diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/container.py b/ix-dev/community/lidarr/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/lidarr/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/deps.py b/ix-dev/community/lidarr/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/lidarr/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/devices.py b/ix-dev/community/lidarr/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/lidarr/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/environment.py b/ix-dev/community/lidarr/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/lidarr/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/functions.py b/ix-dev/community/lidarr/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/lidarr/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/lidarr/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/lidarr/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/storage.py b/ix-dev/community/lidarr/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/lidarr/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/validations.py b/ix-dev/community/lidarr/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/lidarr/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/configs.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/container.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/depends.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/deps.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/device.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/device.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/devices.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/dns.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/environment.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/error.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/error.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/functions.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/labels.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/notes.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/portal.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/portals.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/ports.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/render.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/render.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/resources.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/restart.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/storage.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/lidarr/templates/library/base_v2_1_0/validations.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/lidarr/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/lidarr/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/lidarr/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/lidarr/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/lidarr/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/linkding/app.yaml b/ix-dev/community/linkding/app.yaml index f334d68baa..167cb33569 100644 --- a/ix-dev/community/linkding/app.yaml +++ b/ix-dev/community/linkding/app.yaml @@ -8,8 +8,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/linkding/icons/icon.svg keywords: - bookmark -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -33,4 +33,4 @@ sources: - https://hub.docker.com/r/sissbruecker/linkding/ title: Linkding train: community -version: 1.1.4 +version: 1.1.5 diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/container.py b/ix-dev/community/linkding/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/linkding/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/deps.py b/ix-dev/community/linkding/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/linkding/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/devices.py b/ix-dev/community/linkding/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/linkding/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/environment.py b/ix-dev/community/linkding/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/linkding/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/functions.py b/ix-dev/community/linkding/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/linkding/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/linkding/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/linkding/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/storage.py b/ix-dev/community/linkding/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/linkding/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/validations.py b/ix-dev/community/linkding/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/linkding/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/configs.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/container.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/depends.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/deps.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/device.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/device.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/devices.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/dns.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/environment.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/error.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/error.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/functions.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/labels.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/notes.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/portal.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/portals.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/ports.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/render.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/render.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/resources.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/restart.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/storage.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/linkding/templates/library/base_v2_1_0/validations.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/linkding/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/linkding/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/linkding/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/linkding/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/linkding/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/listmonk/app.yaml b/ix-dev/community/listmonk/app.yaml index db2182f7d0..3475fc0a8a 100644 --- a/ix-dev/community/listmonk/app.yaml +++ b/ix-dev/community/listmonk/app.yaml @@ -19,8 +19,8 @@ icon: https://media.sys.truenas.net/apps/listmonk/icons/icon.svg keywords: - mailing-list - newsletter -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -46,4 +46,4 @@ sources: - https://github.com/knadh/listmonk title: Listmonk train: community -version: 1.1.3 +version: 1.1.4 diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/container.py b/ix-dev/community/listmonk/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/listmonk/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/deps.py b/ix-dev/community/listmonk/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/listmonk/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/devices.py b/ix-dev/community/listmonk/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/listmonk/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/environment.py b/ix-dev/community/listmonk/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/listmonk/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/functions.py b/ix-dev/community/listmonk/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/listmonk/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/listmonk/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/listmonk/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/storage.py b/ix-dev/community/listmonk/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/listmonk/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/validations.py b/ix-dev/community/listmonk/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/listmonk/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/configs.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/container.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/depends.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/deps.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/device.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/device.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/devices.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/dns.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/environment.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/error.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/error.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/functions.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/labels.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/notes.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/portal.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/portals.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/ports.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/render.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/render.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/resources.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/restart.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/storage.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/listmonk/templates/library/base_v2_1_0/validations.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/listmonk/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/listmonk/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/listmonk/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/listmonk/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/listmonk/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/logseq/app.yaml b/ix-dev/community/logseq/app.yaml index 377e3e86d4..5d7b4ff335 100644 --- a/ix-dev/community/logseq/app.yaml +++ b/ix-dev/community/logseq/app.yaml @@ -10,8 +10,8 @@ icon: https://media.sys.truenas.net/apps/logseq/icons/icon.png keywords: - knowledge - management -lib_version: 2.0.27 -lib_version_hash: 5e659d9e9d2ef1170873dab29e34218340f47a9d8391c3d8de2ea8b8662fa063 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -28,4 +28,4 @@ sources: - https://github.com/logseq/logseq title: Logseq train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/container.py b/ix-dev/community/logseq/templates/library/base_v2_0_27/container.py deleted file mode 100644 index 440926dd5b..0000000000 --- a/ix-dev/community/logseq/templates/library/base_v2_0_27/container.py +++ /dev/null @@ -1,313 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/deps.py b/ix-dev/community/logseq/templates/library/base_v2_0_27/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/logseq/templates/library/base_v2_0_27/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/devices.py b/ix-dev/community/logseq/templates/library/base_v2_0_27/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/logseq/templates/library/base_v2_0_27/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/environment.py b/ix-dev/community/logseq/templates/library/base_v2_0_27/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/logseq/templates/library/base_v2_0_27/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/functions.py b/ix-dev/community/logseq/templates/library/base_v2_0_27/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/logseq/templates/library/base_v2_0_27/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/healthcheck.py b/ix-dev/community/logseq/templates/library/base_v2_0_27/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/logseq/templates/library/base_v2_0_27/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_container.py b/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_container.py deleted file mode 100644 index 84d712a403..0000000000 --- a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_container.py +++ /dev/null @@ -1,314 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_deps.py b/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_device.py b/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_environment.py b/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_functions.py b/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_healthcheck.py b/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/validations.py b/ix-dev/community/logseq/templates/library/base_v2_0_27/validations.py deleted file mode 100644 index 155c2e1091..0000000000 --- a/ix-dev/community/logseq/templates/library/base_v2_0_27/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/__init__.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/__init__.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/configs.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/configs.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/container.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/depends.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/depends.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/deploy.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/deploy.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/deps.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/device.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/device.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/devices.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/dns.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/dns.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/environment.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/error.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/error.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/formatter.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/formatter.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/functions.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/labels.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/labels.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/notes.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/notes.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/portal.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/portal.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/portals.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/portals.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/ports.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/ports.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/render.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/render.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/resources.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/resources.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/restart.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/restart.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/storage.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/__init__.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/tests/__init__.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_build_image.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_build_image.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_configs.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_configs.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_depends.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_depends.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_dns.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_dns.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_formatter.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_formatter.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_labels.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_labels.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_notes.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_notes.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_portal.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_portal.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_ports.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_ports.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_render.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_render.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_resources.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_resources.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_restart.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/tests/test_restart.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/logseq/templates/library/base_v2_1_0/validations.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/logseq/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/volume_mount.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/volume_mount.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/volume_mount_types.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/volume_mount_types.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/volume_sources.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/volume_sources.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/volume_types.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/volume_types.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/logseq/templates/library/base_v2_0_27/volumes.py b/ix-dev/community/logseq/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/logseq/templates/library/base_v2_0_27/volumes.py rename to ix-dev/community/logseq/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/mealie/app.yaml b/ix-dev/community/mealie/app.yaml index eb79d558c3..caa48cb862 100644 --- a/ix-dev/community/mealie/app.yaml +++ b/ix-dev/community/mealie/app.yaml @@ -9,8 +9,8 @@ icon: https://media.sys.truenas.net/apps/mealie/icons/icon.png keywords: - recipes - meal planner -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -33,4 +33,4 @@ sources: - https://docs.mealie.io/ title: Mealie train: community -version: 1.3.4 +version: 1.3.5 diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/container.py b/ix-dev/community/mealie/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/mealie/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/deps.py b/ix-dev/community/mealie/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/mealie/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/devices.py b/ix-dev/community/mealie/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/mealie/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/environment.py b/ix-dev/community/mealie/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/mealie/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/functions.py b/ix-dev/community/mealie/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/mealie/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/mealie/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/mealie/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/storage.py b/ix-dev/community/mealie/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/mealie/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/validations.py b/ix-dev/community/mealie/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/mealie/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/configs.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/container.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/depends.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/deps.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/device.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/device.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/devices.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/dns.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/environment.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/error.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/error.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/functions.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/labels.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/notes.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/portal.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/portals.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/ports.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/render.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/render.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/resources.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/restart.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/storage.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/mealie/templates/library/base_v2_1_0/validations.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/mealie/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/mealie/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/mealie/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/mealie/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/mealie/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/metube/app.yaml b/ix-dev/community/metube/app.yaml index 1f92ab17c4..4f39254e28 100644 --- a/ix-dev/community/metube/app.yaml +++ b/ix-dev/community/metube/app.yaml @@ -10,8 +10,8 @@ icon: https://media.sys.truenas.net/apps/metube/icons/icon.svg keywords: - youtube-dl - yt-dlp -lib_version: 2.0.25 -lib_version_hash: 37aa2a7f78f8ef1e8f3d705f4ce5d8bb66eddd4f023c25294a53a893b21a6048 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -30,4 +30,4 @@ sources: - https://github.com/alexta69/metube title: MeTube train: community -version: 1.2.2 +version: 1.2.3 diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/container.py b/ix-dev/community/metube/templates/library/base_v2_0_25/container.py deleted file mode 100644 index 440926dd5b..0000000000 --- a/ix-dev/community/metube/templates/library/base_v2_0_25/container.py +++ /dev/null @@ -1,313 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/deps.py b/ix-dev/community/metube/templates/library/base_v2_0_25/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/metube/templates/library/base_v2_0_25/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/devices.py b/ix-dev/community/metube/templates/library/base_v2_0_25/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/metube/templates/library/base_v2_0_25/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/environment.py b/ix-dev/community/metube/templates/library/base_v2_0_25/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/metube/templates/library/base_v2_0_25/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/functions.py b/ix-dev/community/metube/templates/library/base_v2_0_25/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/metube/templates/library/base_v2_0_25/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/healthcheck.py b/ix-dev/community/metube/templates/library/base_v2_0_25/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/metube/templates/library/base_v2_0_25/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_container.py b/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_container.py deleted file mode 100644 index 84d712a403..0000000000 --- a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_container.py +++ /dev/null @@ -1,314 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_deps.py b/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_device.py b/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_environment.py b/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_functions.py b/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_healthcheck.py b/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/validations.py b/ix-dev/community/metube/templates/library/base_v2_0_25/validations.py deleted file mode 100644 index 155c2e1091..0000000000 --- a/ix-dev/community/metube/templates/library/base_v2_0_25/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/__init__.py b/ix-dev/community/metube/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/__init__.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/configs.py b/ix-dev/community/metube/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/configs.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/container.py b/ix-dev/community/metube/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/depends.py b/ix-dev/community/metube/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/depends.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/deploy.py b/ix-dev/community/metube/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/deploy.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/deps.py b/ix-dev/community/metube/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/metube/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/metube/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/metube/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/metube/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/device.py b/ix-dev/community/metube/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/device.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/devices.py b/ix-dev/community/metube/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/dns.py b/ix-dev/community/metube/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/dns.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/environment.py b/ix-dev/community/metube/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/error.py b/ix-dev/community/metube/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/error.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/formatter.py b/ix-dev/community/metube/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/formatter.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/functions.py b/ix-dev/community/metube/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/metube/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/labels.py b/ix-dev/community/metube/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/labels.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/notes.py b/ix-dev/community/metube/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/notes.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/portal.py b/ix-dev/community/metube/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/portal.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/portals.py b/ix-dev/community/metube/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/portals.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/ports.py b/ix-dev/community/metube/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/ports.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/render.py b/ix-dev/community/metube/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/render.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/resources.py b/ix-dev/community/metube/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/resources.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/restart.py b/ix-dev/community/metube/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/restart.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/storage.py b/ix-dev/community/metube/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/metube/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/__init__.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/tests/__init__.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_build_image.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_build_image.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_configs.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_configs.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_depends.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_depends.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_dns.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_dns.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_formatter.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_formatter.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_labels.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_labels.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_notes.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_notes.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_portal.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_portal.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_ports.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_ports.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_render.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_render.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_resources.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_resources.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_restart.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/tests/test_restart.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/metube/templates/library/base_v2_1_0/validations.py b/ix-dev/community/metube/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/metube/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/volume_mount.py b/ix-dev/community/metube/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/volume_mount.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/volume_mount_types.py b/ix-dev/community/metube/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/volume_mount_types.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/volume_sources.py b/ix-dev/community/metube/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/volume_sources.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/volume_types.py b/ix-dev/community/metube/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/volume_types.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/metube/templates/library/base_v2_0_25/volumes.py b/ix-dev/community/metube/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/metube/templates/library/base_v2_0_25/volumes.py rename to ix-dev/community/metube/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/minecraft/app.yaml b/ix-dev/community/minecraft/app.yaml index 2d76bbd085..d246eb78e7 100644 --- a/ix-dev/community/minecraft/app.yaml +++ b/ix-dev/community/minecraft/app.yaml @@ -19,8 +19,8 @@ icon: https://media.sys.truenas.net/apps/minecraft/icons/icon.svg keywords: - world - building -lib_version: 2.0.26 -lib_version_hash: e82ef6b594774d6160be29cae52fd4f8367518498ea1cd386d7d9edc4ea8b654 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -38,4 +38,4 @@ sources: - https://github.com/itzg/docker-minecraft-server title: Minecraft train: community -version: 1.12.0 +version: 1.12.1 diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/container.py b/ix-dev/community/minecraft/templates/library/base_v2_0_26/container.py deleted file mode 100644 index 440926dd5b..0000000000 --- a/ix-dev/community/minecraft/templates/library/base_v2_0_26/container.py +++ /dev/null @@ -1,313 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/deps.py b/ix-dev/community/minecraft/templates/library/base_v2_0_26/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/minecraft/templates/library/base_v2_0_26/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/devices.py b/ix-dev/community/minecraft/templates/library/base_v2_0_26/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/minecraft/templates/library/base_v2_0_26/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/environment.py b/ix-dev/community/minecraft/templates/library/base_v2_0_26/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/minecraft/templates/library/base_v2_0_26/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/functions.py b/ix-dev/community/minecraft/templates/library/base_v2_0_26/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/minecraft/templates/library/base_v2_0_26/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/healthcheck.py b/ix-dev/community/minecraft/templates/library/base_v2_0_26/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/minecraft/templates/library/base_v2_0_26/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_container.py b/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_container.py deleted file mode 100644 index 84d712a403..0000000000 --- a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_container.py +++ /dev/null @@ -1,314 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_deps.py b/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_device.py b/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_environment.py b/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_functions.py b/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_healthcheck.py b/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/validations.py b/ix-dev/community/minecraft/templates/library/base_v2_0_26/validations.py deleted file mode 100644 index 155c2e1091..0000000000 --- a/ix-dev/community/minecraft/templates/library/base_v2_0_26/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/__init__.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/__init__.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/configs.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/configs.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/container.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/depends.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/depends.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/deploy.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/deploy.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/deps.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/device.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/device.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/devices.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/dns.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/dns.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/environment.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/error.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/error.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/formatter.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/formatter.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/functions.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/labels.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/labels.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/notes.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/notes.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/portal.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/portal.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/portals.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/portals.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/ports.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/ports.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/render.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/render.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/resources.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/resources.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/restart.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/restart.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/storage.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/__init__.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/__init__.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_build_image.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_build_image.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_configs.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_configs.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_depends.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_depends.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_dns.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_dns.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_formatter.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_formatter.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_labels.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_labels.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_notes.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_notes.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_portal.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_portal.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_ports.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_ports.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_render.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_render.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_resources.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_resources.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_restart.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/tests/test_restart.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/minecraft/templates/library/base_v2_1_0/validations.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/minecraft/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/volume_mount.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/volume_mount.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/volume_mount_types.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/volume_mount_types.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/volume_sources.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/volume_sources.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/volume_types.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/volume_types.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/minecraft/templates/library/base_v2_0_26/volumes.py b/ix-dev/community/minecraft/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/minecraft/templates/library/base_v2_0_26/volumes.py rename to ix-dev/community/minecraft/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/mineos/app.yaml b/ix-dev/community/mineos/app.yaml index dd25663d75..58bdbc928d 100644 --- a/ix-dev/community/mineos/app.yaml +++ b/ix-dev/community/mineos/app.yaml @@ -19,8 +19,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/mineos/icons/icon.png keywords: - minecraft -lib_version: 2.0.28 -lib_version_hash: 7126a87f7e27b162098f15622204132e097ca7ff4e6cc7f14b2a4d179c06502b +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -39,4 +39,4 @@ sources: - https://github.com/hexparrot/mineos-node title: MineOS train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/container.py b/ix-dev/community/mineos/templates/library/base_v2_0_28/container.py deleted file mode 100644 index 76d1931570..0000000000 --- a/ix-dev/community/mineos/templates/library/base_v2_0_28/container.py +++ /dev/null @@ -1,313 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/deps.py b/ix-dev/community/mineos/templates/library/base_v2_0_28/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/mineos/templates/library/base_v2_0_28/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/devices.py b/ix-dev/community/mineos/templates/library/base_v2_0_28/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/mineos/templates/library/base_v2_0_28/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/environment.py b/ix-dev/community/mineos/templates/library/base_v2_0_28/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/mineos/templates/library/base_v2_0_28/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/functions.py b/ix-dev/community/mineos/templates/library/base_v2_0_28/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/mineos/templates/library/base_v2_0_28/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/healthcheck.py b/ix-dev/community/mineos/templates/library/base_v2_0_28/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/mineos/templates/library/base_v2_0_28/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_container.py b/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_container.py deleted file mode 100644 index fd0b9f9fff..0000000000 --- a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_container.py +++ /dev/null @@ -1,314 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_deps.py b/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_device.py b/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_environment.py b/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_functions.py b/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_healthcheck.py b/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/validations.py b/ix-dev/community/mineos/templates/library/base_v2_0_28/validations.py deleted file mode 100644 index 155c2e1091..0000000000 --- a/ix-dev/community/mineos/templates/library/base_v2_0_28/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/__init__.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/__init__.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/configs.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/configs.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/container.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/depends.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/depends.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/deploy.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/deploy.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/deps.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/device.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/device.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/devices.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/dns.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/dns.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/environment.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/error.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/error.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/formatter.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/formatter.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/functions.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/labels.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/labels.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/notes.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/notes.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/portal.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/portal.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/portals.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/portals.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/ports.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/ports.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/render.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/render.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/resources.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/resources.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/restart.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/restart.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/storage.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/__init__.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/tests/__init__.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_build_image.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_build_image.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_configs.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_configs.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_depends.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_depends.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_dns.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_dns.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_formatter.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_formatter.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_labels.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_labels.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_notes.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_notes.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_portal.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_portal.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_ports.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_ports.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_render.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_render.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_resources.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_resources.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_restart.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/tests/test_restart.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/mineos/templates/library/base_v2_1_0/validations.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/mineos/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/volume_mount.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/volume_mount.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/volume_mount_types.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/volume_mount_types.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/volume_sources.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/volume_sources.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/volume_types.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/volume_types.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/mineos/templates/library/base_v2_0_28/volumes.py b/ix-dev/community/mineos/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/mineos/templates/library/base_v2_0_28/volumes.py rename to ix-dev/community/mineos/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/mumble/app.yaml b/ix-dev/community/mumble/app.yaml index 983aabc0b3..60861df458 100644 --- a/ix-dev/community/mumble/app.yaml +++ b/ix-dev/community/mumble/app.yaml @@ -8,8 +8,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/mumble/icons/icon.svg keywords: - voice -lib_version: 2.0.26 -lib_version_hash: e82ef6b594774d6160be29cae52fd4f8367518498ea1cd386d7d9edc4ea8b654 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -27,4 +27,4 @@ sources: - https://www.mumble.info/ title: Mumble train: community -version: 1.2.0 +version: 1.2.1 diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/container.py b/ix-dev/community/mumble/templates/library/base_v2_0_26/container.py deleted file mode 100644 index 440926dd5b..0000000000 --- a/ix-dev/community/mumble/templates/library/base_v2_0_26/container.py +++ /dev/null @@ -1,313 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/deps.py b/ix-dev/community/mumble/templates/library/base_v2_0_26/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/mumble/templates/library/base_v2_0_26/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/devices.py b/ix-dev/community/mumble/templates/library/base_v2_0_26/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/mumble/templates/library/base_v2_0_26/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/environment.py b/ix-dev/community/mumble/templates/library/base_v2_0_26/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/mumble/templates/library/base_v2_0_26/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/functions.py b/ix-dev/community/mumble/templates/library/base_v2_0_26/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/mumble/templates/library/base_v2_0_26/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/healthcheck.py b/ix-dev/community/mumble/templates/library/base_v2_0_26/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/mumble/templates/library/base_v2_0_26/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_container.py b/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_container.py deleted file mode 100644 index 84d712a403..0000000000 --- a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_container.py +++ /dev/null @@ -1,314 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_deps.py b/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_device.py b/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_environment.py b/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_functions.py b/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_healthcheck.py b/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/validations.py b/ix-dev/community/mumble/templates/library/base_v2_0_26/validations.py deleted file mode 100644 index 155c2e1091..0000000000 --- a/ix-dev/community/mumble/templates/library/base_v2_0_26/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/__init__.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/__init__.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/configs.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/configs.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/container.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/depends.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/depends.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/deploy.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/deploy.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/deps.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/device.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/device.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/devices.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/dns.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/dns.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/environment.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/error.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/error.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/formatter.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/formatter.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/functions.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/labels.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/labels.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/notes.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/notes.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/portal.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/portal.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/portals.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/portals.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/ports.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/ports.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/render.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/render.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/resources.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/resources.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/restart.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/restart.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/storage.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/__init__.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/tests/__init__.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_build_image.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_build_image.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_configs.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_configs.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_depends.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_depends.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_dns.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_dns.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_formatter.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_formatter.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_labels.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_labels.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_notes.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_notes.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_portal.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_portal.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_ports.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_ports.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_render.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_render.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_resources.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_resources.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_restart.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/tests/test_restart.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/mumble/templates/library/base_v2_1_0/validations.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/mumble/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/volume_mount.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/volume_mount.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/volume_mount_types.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/volume_mount_types.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/volume_sources.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/volume_sources.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/volume_types.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/volume_types.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/mumble/templates/library/base_v2_0_26/volumes.py b/ix-dev/community/mumble/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/mumble/templates/library/base_v2_0_26/volumes.py rename to ix-dev/community/mumble/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/n8n/app.yaml b/ix-dev/community/n8n/app.yaml index c859dff67b..9121caaa18 100644 --- a/ix-dev/community/n8n/app.yaml +++ b/ix-dev/community/n8n/app.yaml @@ -9,8 +9,8 @@ icon: https://media.sys.truenas.net/apps/n8n/icons/icon.png keywords: - workflows - automation -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -42,4 +42,4 @@ sources: - https://hub.docker.com/r/n8nio/n8n title: n8n train: community -version: 1.4.11 +version: 1.4.12 diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/container.py b/ix-dev/community/n8n/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/n8n/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/deps.py b/ix-dev/community/n8n/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/n8n/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/devices.py b/ix-dev/community/n8n/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/n8n/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/environment.py b/ix-dev/community/n8n/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/n8n/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/functions.py b/ix-dev/community/n8n/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/n8n/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/n8n/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/n8n/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/storage.py b/ix-dev/community/n8n/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/n8n/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/validations.py b/ix-dev/community/n8n/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/n8n/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/configs.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/container.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/depends.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/deps.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/device.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/device.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/devices.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/dns.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/environment.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/error.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/error.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/functions.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/labels.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/notes.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/portal.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/portals.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/ports.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/render.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/render.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/resources.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/restart.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/storage.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/n8n/templates/library/base_v2_1_0/validations.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/n8n/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/n8n/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/n8n/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/n8n/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/n8n/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/navidrome/app.yaml b/ix-dev/community/navidrome/app.yaml index 62e1e3af18..79eb675def 100644 --- a/ix-dev/community/navidrome/app.yaml +++ b/ix-dev/community/navidrome/app.yaml @@ -11,8 +11,8 @@ icon: https://media.sys.truenas.net/apps/navidrome/icons/icon.png keywords: - media - music -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -32,4 +32,4 @@ sources: - https://github.com/navidrome/navidrome/ title: Navidrome train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/container.py b/ix-dev/community/navidrome/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/navidrome/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/deps.py b/ix-dev/community/navidrome/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/navidrome/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/devices.py b/ix-dev/community/navidrome/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/navidrome/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/environment.py b/ix-dev/community/navidrome/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/navidrome/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/functions.py b/ix-dev/community/navidrome/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/navidrome/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/navidrome/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/navidrome/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/validations.py b/ix-dev/community/navidrome/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/navidrome/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/configs.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/container.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/depends.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/deps.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/device.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/device.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/devices.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/dns.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/environment.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/error.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/error.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/functions.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/labels.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/notes.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/portal.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/portals.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/ports.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/render.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/render.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/resources.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/restart.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/storage.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/navidrome/templates/library/base_v2_1_0/validations.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/navidrome/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/navidrome/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/navidrome/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/navidrome/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/navidrome/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/netbootxyz/app.yaml b/ix-dev/community/netbootxyz/app.yaml index 53a0a58380..f6dd3615b7 100644 --- a/ix-dev/community/netbootxyz/app.yaml +++ b/ix-dev/community/netbootxyz/app.yaml @@ -30,8 +30,8 @@ keywords: - netboot - netbootxyz - netboot.xyz -lib_version: 2.0.26 -lib_version_hash: e82ef6b594774d6160be29cae52fd4f8367518498ea1cd386d7d9edc4ea8b654 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -50,4 +50,4 @@ sources: - https://netboot.xyz title: Netboot.xyz train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/container.py b/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/container.py deleted file mode 100644 index 440926dd5b..0000000000 --- a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/container.py +++ /dev/null @@ -1,313 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/deps.py b/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/devices.py b/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/environment.py b/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/functions.py b/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/healthcheck.py b/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_container.py b/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_container.py deleted file mode 100644 index 84d712a403..0000000000 --- a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_container.py +++ /dev/null @@ -1,314 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_deps.py b/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_device.py b/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_environment.py b/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_functions.py b/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_healthcheck.py b/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/validations.py b/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/validations.py deleted file mode 100644 index 155c2e1091..0000000000 --- a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/__init__.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/__init__.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/configs.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/configs.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/container.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/depends.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/depends.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/deploy.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/deploy.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/deps.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/device.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/device.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/devices.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/dns.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/dns.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/environment.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/error.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/error.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/formatter.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/formatter.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/functions.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/labels.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/labels.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/notes.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/notes.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/portal.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/portal.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/portals.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/portals.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/ports.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/ports.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/render.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/render.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/resources.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/resources.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/restart.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/restart.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/storage.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/__init__.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/__init__.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_build_image.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_build_image.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_configs.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_configs.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_depends.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_depends.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_dns.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_dns.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_formatter.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_formatter.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_labels.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_labels.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_notes.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_notes.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_portal.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_portal.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_ports.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_ports.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_render.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_render.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_resources.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_resources.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_restart.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/tests/test_restart.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/validations.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/volume_mount.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/volume_mount.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/volume_mount_types.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/volume_mount_types.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/volume_sources.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/volume_sources.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/volume_types.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/volume_types.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/netbootxyz/templates/library/base_v2_0_26/volumes.py b/ix-dev/community/netbootxyz/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/netbootxyz/templates/library/base_v2_0_26/volumes.py rename to ix-dev/community/netbootxyz/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/nginx-proxy-manager/app.yaml b/ix-dev/community/nginx-proxy-manager/app.yaml index 1f5f2f0028..85f543ee21 100644 --- a/ix-dev/community/nginx-proxy-manager/app.yaml +++ b/ix-dev/community/nginx-proxy-manager/app.yaml @@ -22,8 +22,8 @@ keywords: - reverse - nginx - proxy -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -44,4 +44,4 @@ sources: - https://hub.docker.com/r/jc21/nginx-proxy-manager title: Nginx Proxy Manager train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/container.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/deps.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/devices.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/environment.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/functions.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/validations.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/configs.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/container.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/depends.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/deps.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/device.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/device.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/devices.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/dns.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/environment.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/error.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/error.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/functions.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/labels.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/notes.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/portal.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/portals.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/ports.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/render.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/render.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/resources.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/restart.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/storage.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/validations.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/nginx-proxy-manager/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/nginx-proxy-manager/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/node-red/app.yaml b/ix-dev/community/node-red/app.yaml index 55fb60a63b..0a61adef57 100644 --- a/ix-dev/community/node-red/app.yaml +++ b/ix-dev/community/node-red/app.yaml @@ -9,8 +9,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/node-red/icons/icon.png keywords: - automation -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -29,4 +29,4 @@ sources: - https://github.com/node-red/node-red-docker title: Node-RED train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/container.py b/ix-dev/community/node-red/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/node-red/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/deps.py b/ix-dev/community/node-red/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/node-red/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/devices.py b/ix-dev/community/node-red/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/node-red/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/environment.py b/ix-dev/community/node-red/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/node-red/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/functions.py b/ix-dev/community/node-red/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/node-red/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/node-red/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/node-red/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/validations.py b/ix-dev/community/node-red/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/node-red/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/configs.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/container.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/depends.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/deps.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/device.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/device.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/devices.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/dns.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/environment.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/error.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/error.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/functions.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/labels.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/notes.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/portal.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/portals.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/ports.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/render.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/render.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/resources.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/restart.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/storage.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/node-red/templates/library/base_v2_1_0/validations.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/node-red/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/node-red/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/node-red/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/node-red/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/node-red/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/odoo/app.yaml b/ix-dev/community/odoo/app.yaml index e862a8a05d..43e2e41a74 100644 --- a/ix-dev/community/odoo/app.yaml +++ b/ix-dev/community/odoo/app.yaml @@ -9,8 +9,8 @@ icon: https://media.sys.truenas.net/apps/odoo/icons/icon.png keywords: - erp - odoo -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -35,4 +35,4 @@ sources: - https://github.com/odoo/odoo title: Odoo train: community -version: 1.1.2 +version: 1.1.3 diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/container.py b/ix-dev/community/odoo/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/odoo/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/deps.py b/ix-dev/community/odoo/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/odoo/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/devices.py b/ix-dev/community/odoo/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/odoo/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/environment.py b/ix-dev/community/odoo/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/odoo/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/functions.py b/ix-dev/community/odoo/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/odoo/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/odoo/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/odoo/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/storage.py b/ix-dev/community/odoo/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/odoo/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/validations.py b/ix-dev/community/odoo/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/odoo/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/configs.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/container.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/depends.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/deps.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/device.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/device.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/devices.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/dns.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/environment.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/error.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/error.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/functions.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/labels.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/notes.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/portal.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/portals.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/ports.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/render.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/render.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/resources.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/restart.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/storage.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/odoo/templates/library/base_v2_1_0/validations.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/odoo/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/odoo/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/odoo/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/odoo/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/odoo/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/ollama/app.yaml b/ix-dev/community/ollama/app.yaml index 290904f378..e327421c9d 100644 --- a/ix-dev/community/ollama/app.yaml +++ b/ix-dev/community/ollama/app.yaml @@ -10,8 +10,8 @@ icon: https://media.sys.truenas.net/apps/ollama/icons/icon.png keywords: - ai - llm -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -28,4 +28,4 @@ sources: - https://github.com/ollama/ollama title: Ollama train: community -version: 1.0.16 +version: 1.0.17 diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/container.py b/ix-dev/community/ollama/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/ollama/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/deps.py b/ix-dev/community/ollama/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/ollama/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/devices.py b/ix-dev/community/ollama/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/ollama/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/environment.py b/ix-dev/community/ollama/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/ollama/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/functions.py b/ix-dev/community/ollama/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/ollama/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/ollama/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/ollama/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/storage.py b/ix-dev/community/ollama/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/ollama/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/validations.py b/ix-dev/community/ollama/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/ollama/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/configs.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/container.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/depends.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/deps.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/device.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/device.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/devices.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/dns.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/environment.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/error.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/error.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/functions.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/labels.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/notes.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/portal.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/portals.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/ports.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/render.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/render.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/resources.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/restart.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/storage.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/ollama/templates/library/base_v2_1_0/validations.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/ollama/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/ollama/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/ollama/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/ollama/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/ollama/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/omada-controller/app.yaml b/ix-dev/community/omada-controller/app.yaml index 9f2405f24b..becdd5ff4f 100644 --- a/ix-dev/community/omada-controller/app.yaml +++ b/ix-dev/community/omada-controller/app.yaml @@ -23,8 +23,8 @@ keywords: - controller - omada - tp-link -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -42,4 +42,4 @@ sources: - https://hub.docker.com/r/mbentley/omada-controller title: Omada Controller train: community -version: 1.2.3 +version: 1.2.4 diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/container.py b/ix-dev/community/omada-controller/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/deps.py b/ix-dev/community/omada-controller/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/devices.py b/ix-dev/community/omada-controller/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/environment.py b/ix-dev/community/omada-controller/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/functions.py b/ix-dev/community/omada-controller/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/omada-controller/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/storage.py b/ix-dev/community/omada-controller/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/validations.py b/ix-dev/community/omada-controller/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/configs.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/container.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/depends.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/deps.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/device.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/device.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/devices.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/dns.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/environment.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/error.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/error.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/functions.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/labels.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/notes.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/portal.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/portals.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/ports.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/render.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/render.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/resources.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/restart.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/storage.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_1_0/validations.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/omada-controller/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/omada-controller/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/omada-controller/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/omada-controller/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/open-webui/app.yaml b/ix-dev/community/open-webui/app.yaml index d037a27be5..9bf9d32bd4 100644 --- a/ix-dev/community/open-webui/app.yaml +++ b/ix-dev/community/open-webui/app.yaml @@ -13,8 +13,8 @@ keywords: - llm - webui - open-webui -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -32,4 +32,4 @@ sources: - https://github.com/open-webui/open-webui title: Open WebUI train: community -version: 1.0.13 +version: 1.0.14 diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/container.py b/ix-dev/community/open-webui/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/open-webui/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/deps.py b/ix-dev/community/open-webui/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/open-webui/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/devices.py b/ix-dev/community/open-webui/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/open-webui/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/environment.py b/ix-dev/community/open-webui/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/open-webui/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/functions.py b/ix-dev/community/open-webui/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/open-webui/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/open-webui/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/open-webui/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/storage.py b/ix-dev/community/open-webui/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/open-webui/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/validations.py b/ix-dev/community/open-webui/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/open-webui/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/configs.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/container.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/depends.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/deps.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/device.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/device.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/devices.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/dns.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/environment.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/error.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/error.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/functions.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/labels.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/notes.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/portal.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/portals.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/ports.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/render.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/render.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/resources.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/restart.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/storage.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/open-webui/templates/library/base_v2_1_0/validations.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/open-webui/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/open-webui/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/open-webui/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/open-webui/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/open-webui/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/organizr/app.yaml b/ix-dev/community/organizr/app.yaml index f5291aecc6..a792824df6 100644 --- a/ix-dev/community/organizr/app.yaml +++ b/ix-dev/community/organizr/app.yaml @@ -19,8 +19,8 @@ icon: https://media.sys.truenas.net/apps/organizr/icons/icon.png keywords: - dashboard - organizr -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -40,4 +40,4 @@ sources: - https://github.com/causefx/Organizr title: Organizr train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/container.py b/ix-dev/community/organizr/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/organizr/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/deps.py b/ix-dev/community/organizr/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/organizr/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/devices.py b/ix-dev/community/organizr/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/organizr/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/environment.py b/ix-dev/community/organizr/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/organizr/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/functions.py b/ix-dev/community/organizr/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/organizr/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/organizr/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/organizr/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/validations.py b/ix-dev/community/organizr/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/organizr/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/configs.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/container.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/depends.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/deps.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/device.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/device.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/devices.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/dns.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/environment.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/error.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/error.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/functions.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/labels.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/notes.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/portal.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/portals.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/ports.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/render.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/render.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/resources.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/restart.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/storage.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/organizr/templates/library/base_v2_1_0/validations.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/organizr/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/organizr/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/organizr/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/organizr/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/organizr/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/overseerr/app.yaml b/ix-dev/community/overseerr/app.yaml index 05c33615e1..6ba9fd4b90 100644 --- a/ix-dev/community/overseerr/app.yaml +++ b/ix-dev/community/overseerr/app.yaml @@ -9,8 +9,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/overseerr/icons/icon.svg keywords: - media -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -28,4 +28,4 @@ sources: - https://github.com/sct/overseerr title: Overseerr train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/container.py b/ix-dev/community/overseerr/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/overseerr/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/deps.py b/ix-dev/community/overseerr/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/overseerr/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/devices.py b/ix-dev/community/overseerr/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/overseerr/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/environment.py b/ix-dev/community/overseerr/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/overseerr/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/functions.py b/ix-dev/community/overseerr/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/overseerr/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/overseerr/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/overseerr/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/validations.py b/ix-dev/community/overseerr/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/overseerr/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/configs.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/container.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/depends.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/deps.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/device.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/device.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/devices.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/dns.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/environment.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/error.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/error.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/functions.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/labels.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/notes.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/portal.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/portals.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/ports.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/render.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/render.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/resources.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/restart.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/storage.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/overseerr/templates/library/base_v2_1_0/validations.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/overseerr/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/overseerr/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/overseerr/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/overseerr/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/overseerr/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/palworld/app.yaml b/ix-dev/community/palworld/app.yaml index d5db870d63..9426755268 100644 --- a/ix-dev/community/palworld/app.yaml +++ b/ix-dev/community/palworld/app.yaml @@ -28,8 +28,8 @@ icon: https://media.sys.truenas.net/apps/palworld/icons/icon.webp keywords: - game - palworld -lib_version: 2.0.30 -lib_version_hash: a87750c752394f7bf0eb461678564a9ef99bc4e4541787ef25e7be1095a6f879 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -47,4 +47,4 @@ sources: - https://github.com/ich777/docker-steamcmd-server/tree/palworld title: Palworld train: community -version: 1.1.2 +version: 1.1.3 diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/container.py b/ix-dev/community/palworld/templates/library/base_v2_0_30/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/palworld/templates/library/base_v2_0_30/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/deps.py b/ix-dev/community/palworld/templates/library/base_v2_0_30/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/palworld/templates/library/base_v2_0_30/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/devices.py b/ix-dev/community/palworld/templates/library/base_v2_0_30/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/palworld/templates/library/base_v2_0_30/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/environment.py b/ix-dev/community/palworld/templates/library/base_v2_0_30/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/palworld/templates/library/base_v2_0_30/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/functions.py b/ix-dev/community/palworld/templates/library/base_v2_0_30/functions.py deleted file mode 100644 index b0c834ab4b..0000000000 --- a/ix-dev/community/palworld/templates/library/base_v2_0_30/functions.py +++ /dev/null @@ -1,142 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _or_default(self, value, default): - if not value: - return default - return value - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - "or_default": self._or_default, - } diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/healthcheck.py b/ix-dev/community/palworld/templates/library/base_v2_0_30/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/palworld/templates/library/base_v2_0_30/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_container.py b/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_deps.py b/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_device.py b/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_environment.py b/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_functions.py b/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_functions.py deleted file mode 100644 index 13d5e49522..0000000000 --- a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_functions.py +++ /dev/null @@ -1,82 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - {"func": "or_default", "values": [None, 1], "expected": 1}, - {"func": "or_default", "values": [1, None], "expected": 1}, - {"func": "or_default", "values": [False, 1], "expected": 1}, - {"func": "or_default", "values": [True, 1], "expected": True}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_healthcheck.py b/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/validations.py b/ix-dev/community/palworld/templates/library/base_v2_0_30/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/palworld/templates/library/base_v2_0_30/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/__init__.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/__init__.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/configs.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/configs.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/container.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/depends.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/depends.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/deploy.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/deploy.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/deps.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/device.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/device.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/devices.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/dns.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/dns.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/environment.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/error.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/error.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/formatter.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/formatter.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/functions.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/labels.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/labels.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/notes.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/notes.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/portal.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/portal.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/portals.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/portals.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/ports.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/ports.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/render.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/render.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/resources.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/resources.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/restart.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/restart.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/storage.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/__init__.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/tests/__init__.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_build_image.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_build_image.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_configs.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_configs.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_depends.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_depends.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_dns.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_dns.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_formatter.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_formatter.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_labels.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_labels.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_notes.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_notes.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_portal.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_portal.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_ports.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_ports.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_render.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_render.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_resources.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_resources.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_restart.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/tests/test_restart.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/palworld/templates/library/base_v2_1_0/validations.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/palworld/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/volume_mount.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/volume_mount.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/volume_mount_types.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/volume_mount_types.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/volume_sources.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/volume_sources.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/volume_types.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/volume_types.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/palworld/templates/library/base_v2_0_30/volumes.py b/ix-dev/community/palworld/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/palworld/templates/library/base_v2_0_30/volumes.py rename to ix-dev/community/palworld/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/paperless-ngx/app.yaml b/ix-dev/community/paperless-ngx/app.yaml index 11a2492fef..4fdf290a86 100644 --- a/ix-dev/community/paperless-ngx/app.yaml +++ b/ix-dev/community/paperless-ngx/app.yaml @@ -20,8 +20,8 @@ icon: https://media.sys.truenas.net/apps/paperless-ngx/icons/icon.svg keywords: - document - management -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -67,4 +67,4 @@ sources: - https://github.com/paperless-ngx/paperless-ngx title: Paperless-ngx train: community -version: 1.1.7 +version: 1.1.8 diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/container.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/deps.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/devices.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/environment.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/functions.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/storage.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/validations.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/configs.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/container.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/depends.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/deps.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/device.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/device.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/devices.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/dns.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/environment.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/error.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/error.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/functions.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/labels.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/notes.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/portal.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/portals.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/ports.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/render.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/render.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/resources.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/restart.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/storage.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/validations.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/paperless-ngx/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/paperless-ngx/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/passbolt/app.yaml b/ix-dev/community/passbolt/app.yaml index 2cdfe41f05..f0d17aa784 100644 --- a/ix-dev/community/passbolt/app.yaml +++ b/ix-dev/community/passbolt/app.yaml @@ -9,8 +9,8 @@ icon: https://media.sys.truenas.net/apps/passbolt/icons/icon.svg keywords: - password - manager -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -37,4 +37,4 @@ sources: - https://www.passbolt.com title: Passbolt train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/container.py b/ix-dev/community/passbolt/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/passbolt/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/deps.py b/ix-dev/community/passbolt/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/passbolt/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/devices.py b/ix-dev/community/passbolt/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/passbolt/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/environment.py b/ix-dev/community/passbolt/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/passbolt/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/functions.py b/ix-dev/community/passbolt/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/passbolt/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/passbolt/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/passbolt/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/validations.py b/ix-dev/community/passbolt/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/passbolt/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/configs.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/container.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/depends.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/deps.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/device.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/device.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/devices.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/dns.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/environment.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/error.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/error.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/functions.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/labels.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/notes.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/portal.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/portals.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/ports.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/render.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/render.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/resources.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/restart.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/storage.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/passbolt/templates/library/base_v2_1_0/validations.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/passbolt/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/passbolt/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/passbolt/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/passbolt/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/passbolt/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/penpot/app.yaml b/ix-dev/community/penpot/app.yaml index 1949f2431f..b430ba54e0 100644 --- a/ix-dev/community/penpot/app.yaml +++ b/ix-dev/community/penpot/app.yaml @@ -18,8 +18,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/penpot/icons/icon.svg keywords: - design -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -52,4 +52,4 @@ sources: - https://github.com/penpot/penpot title: Penpot train: community -version: 1.0.12 +version: 1.0.13 diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/container.py b/ix-dev/community/penpot/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/penpot/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/deps.py b/ix-dev/community/penpot/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/penpot/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/devices.py b/ix-dev/community/penpot/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/penpot/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/environment.py b/ix-dev/community/penpot/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/penpot/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/functions.py b/ix-dev/community/penpot/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/penpot/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/penpot/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/penpot/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/storage.py b/ix-dev/community/penpot/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/penpot/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/validations.py b/ix-dev/community/penpot/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/penpot/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/configs.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/container.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/depends.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/deps.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/device.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/device.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/devices.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/dns.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/environment.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/error.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/error.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/functions.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/labels.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/notes.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/portal.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/portals.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/ports.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/render.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/render.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/resources.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/restart.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/storage.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/penpot/templates/library/base_v2_1_0/validations.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/penpot/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/penpot/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/penpot/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/penpot/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/penpot/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/pgadmin/app.yaml b/ix-dev/community/pgadmin/app.yaml index 9b19d0a68d..7cbc8ee160 100644 --- a/ix-dev/community/pgadmin/app.yaml +++ b/ix-dev/community/pgadmin/app.yaml @@ -12,8 +12,8 @@ icon: https://media.sys.truenas.net/apps/pgadmin/icons/icon.png keywords: - database - management -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -34,4 +34,4 @@ sources: - https://www.pgadmin.org/ title: pgAdmin train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/container.py b/ix-dev/community/pgadmin/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/deps.py b/ix-dev/community/pgadmin/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/devices.py b/ix-dev/community/pgadmin/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/environment.py b/ix-dev/community/pgadmin/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/functions.py b/ix-dev/community/pgadmin/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/pgadmin/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/validations.py b/ix-dev/community/pgadmin/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/configs.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/container.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/depends.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/deps.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/device.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/device.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/devices.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/dns.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/environment.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/error.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/error.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/functions.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/labels.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/notes.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/portal.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/portals.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/ports.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/render.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/render.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/resources.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/restart.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/storage.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_1_0/validations.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/pgadmin/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/pgadmin/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/pgadmin/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/pgadmin/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/pigallery2/app.yaml b/ix-dev/community/pigallery2/app.yaml index 9af9f633e2..28b0fec05f 100644 --- a/ix-dev/community/pigallery2/app.yaml +++ b/ix-dev/community/pigallery2/app.yaml @@ -10,8 +10,8 @@ icon: https://media.sys.truenas.net/apps/pigallery2/icons/icon.png keywords: - photo - media -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -39,4 +39,4 @@ sources: - https://github.com/bpatrik/pigallery2 title: PiGallery2 train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/container.py b/ix-dev/community/pigallery2/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/deps.py b/ix-dev/community/pigallery2/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/devices.py b/ix-dev/community/pigallery2/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/environment.py b/ix-dev/community/pigallery2/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/functions.py b/ix-dev/community/pigallery2/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/pigallery2/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/validations.py b/ix-dev/community/pigallery2/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/configs.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/container.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/depends.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/deps.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/device.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/device.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/devices.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/dns.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/environment.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/error.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/error.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/functions.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/labels.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/notes.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/portal.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/portals.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/ports.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/render.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/render.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/resources.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/restart.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/storage.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_1_0/validations.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/pigallery2/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/pigallery2/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/pigallery2/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/pigallery2/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/piwigo/app.yaml b/ix-dev/community/piwigo/app.yaml index 1da499693f..15f010c052 100644 --- a/ix-dev/community/piwigo/app.yaml +++ b/ix-dev/community/piwigo/app.yaml @@ -22,8 +22,8 @@ icon: https://media.sys.truenas.net/apps/piwigo/icons/icon.svg keywords: - photo - gallery -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -47,4 +47,4 @@ sources: - https://hub.docker.com/r/linuxserver/piwigo title: Piwigo train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/container.py b/ix-dev/community/piwigo/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/piwigo/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/deps.py b/ix-dev/community/piwigo/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/piwigo/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/devices.py b/ix-dev/community/piwigo/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/piwigo/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/environment.py b/ix-dev/community/piwigo/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/piwigo/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/functions.py b/ix-dev/community/piwigo/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/piwigo/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/piwigo/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/piwigo/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/validations.py b/ix-dev/community/piwigo/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/piwigo/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/configs.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/container.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/depends.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/deps.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/device.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/device.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/devices.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/dns.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/environment.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/error.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/error.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/functions.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/labels.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/notes.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/portal.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/portals.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/ports.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/render.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/render.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/resources.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/restart.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/storage.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/piwigo/templates/library/base_v2_1_0/validations.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/piwigo/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/piwigo/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/piwigo/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/piwigo/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/piwigo/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/planka/app.yaml b/ix-dev/community/planka/app.yaml index 19b3a34320..734ac56df3 100644 --- a/ix-dev/community/planka/app.yaml +++ b/ix-dev/community/planka/app.yaml @@ -10,8 +10,8 @@ keywords: - kanban - project - task -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -35,4 +35,4 @@ sources: - https://github.com/plankanban/planka title: Planka train: community -version: 1.1.6 +version: 1.1.7 diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/container.py b/ix-dev/community/planka/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/planka/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/deps.py b/ix-dev/community/planka/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/planka/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/devices.py b/ix-dev/community/planka/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/planka/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/environment.py b/ix-dev/community/planka/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/planka/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/functions.py b/ix-dev/community/planka/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/planka/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/planka/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/planka/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/storage.py b/ix-dev/community/planka/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/planka/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/validations.py b/ix-dev/community/planka/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/planka/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/planka/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/configs.py b/ix-dev/community/planka/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/container.py b/ix-dev/community/planka/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/depends.py b/ix-dev/community/planka/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/planka/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/deps.py b/ix-dev/community/planka/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/planka/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/planka/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/planka/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/planka/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/device.py b/ix-dev/community/planka/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/device.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/devices.py b/ix-dev/community/planka/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/dns.py b/ix-dev/community/planka/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/environment.py b/ix-dev/community/planka/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/error.py b/ix-dev/community/planka/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/error.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/planka/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/functions.py b/ix-dev/community/planka/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/planka/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/labels.py b/ix-dev/community/planka/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/notes.py b/ix-dev/community/planka/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/portal.py b/ix-dev/community/planka/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/portals.py b/ix-dev/community/planka/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/ports.py b/ix-dev/community/planka/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/render.py b/ix-dev/community/planka/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/render.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/resources.py b/ix-dev/community/planka/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/restart.py b/ix-dev/community/planka/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/storage.py b/ix-dev/community/planka/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/planka/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/planka/templates/library/base_v2_1_0/validations.py b/ix-dev/community/planka/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/planka/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/planka/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/planka/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/planka/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/planka/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/planka/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/planka/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/planka/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/planka/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/plex-auto-languages/app.yaml b/ix-dev/community/plex-auto-languages/app.yaml index fd7ac0f4ab..cef93fe6aa 100644 --- a/ix-dev/community/plex-auto-languages/app.yaml +++ b/ix-dev/community/plex-auto-languages/app.yaml @@ -9,8 +9,8 @@ icon: https://media.sys.truenas.net/apps/plex-auto-languages/icons/icon.svg keywords: - plex - languages -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -27,4 +27,4 @@ sources: - https://github.com/JourneyDocker/Plex-Auto-Languages title: Plex Auto Languages train: community -version: 1.2.0 +version: 1.2.1 diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/container.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/deps.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/devices.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/environment.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/functions.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/validations.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/configs.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/container.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/depends.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/deps.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/device.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/device.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/devices.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/dns.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/environment.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/error.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/error.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/functions.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/labels.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/notes.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/portal.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/portals.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/ports.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/render.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/render.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/resources.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/restart.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/storage.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/validations.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/plex-auto-languages/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/plex-auto-languages/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/portainer/app.yaml b/ix-dev/community/portainer/app.yaml index 7ae6909fb1..ec9eb3fd5e 100644 --- a/ix-dev/community/portainer/app.yaml +++ b/ix-dev/community/portainer/app.yaml @@ -28,8 +28,8 @@ keywords: - docker - compose - container -lib_version: 2.0.30 -lib_version_hash: a87750c752394f7bf0eb461678564a9ef99bc4e4541787ef25e7be1095a6f879 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -49,4 +49,4 @@ sources: - https://github.com/portainer/portainer title: Portainer train: community -version: 1.3.2 +version: 1.3.3 diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/container.py b/ix-dev/community/portainer/templates/library/base_v2_0_30/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/portainer/templates/library/base_v2_0_30/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/deps.py b/ix-dev/community/portainer/templates/library/base_v2_0_30/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/portainer/templates/library/base_v2_0_30/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/devices.py b/ix-dev/community/portainer/templates/library/base_v2_0_30/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/portainer/templates/library/base_v2_0_30/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/environment.py b/ix-dev/community/portainer/templates/library/base_v2_0_30/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/portainer/templates/library/base_v2_0_30/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/functions.py b/ix-dev/community/portainer/templates/library/base_v2_0_30/functions.py deleted file mode 100644 index b0c834ab4b..0000000000 --- a/ix-dev/community/portainer/templates/library/base_v2_0_30/functions.py +++ /dev/null @@ -1,142 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _or_default(self, value, default): - if not value: - return default - return value - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - "or_default": self._or_default, - } diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/healthcheck.py b/ix-dev/community/portainer/templates/library/base_v2_0_30/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/portainer/templates/library/base_v2_0_30/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_container.py b/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_deps.py b/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_device.py b/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_environment.py b/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_functions.py b/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_functions.py deleted file mode 100644 index 13d5e49522..0000000000 --- a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_functions.py +++ /dev/null @@ -1,82 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - {"func": "or_default", "values": [None, 1], "expected": 1}, - {"func": "or_default", "values": [1, None], "expected": 1}, - {"func": "or_default", "values": [False, 1], "expected": 1}, - {"func": "or_default", "values": [True, 1], "expected": True}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_healthcheck.py b/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/validations.py b/ix-dev/community/portainer/templates/library/base_v2_0_30/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/portainer/templates/library/base_v2_0_30/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/__init__.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/__init__.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/configs.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/configs.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/container.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/depends.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/depends.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/deploy.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/deploy.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/deps.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/device.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/device.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/devices.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/dns.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/dns.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/environment.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/error.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/error.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/formatter.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/formatter.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/functions.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/labels.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/labels.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/notes.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/notes.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/portal.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/portal.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/portals.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/portals.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/ports.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/ports.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/render.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/render.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/resources.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/resources.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/restart.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/restart.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/storage.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/__init__.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/tests/__init__.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_build_image.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_build_image.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_configs.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_configs.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_depends.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_depends.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_dns.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_dns.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_formatter.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_formatter.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_labels.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_labels.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_notes.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_notes.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_portal.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_portal.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_ports.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_ports.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_render.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_render.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_resources.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_resources.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_restart.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/tests/test_restart.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/portainer/templates/library/base_v2_1_0/validations.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/portainer/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/volume_mount.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/volume_mount.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/volume_mount_types.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/volume_mount_types.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/volume_sources.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/volume_sources.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/volume_types.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/volume_types.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/portainer/templates/library/base_v2_0_30/volumes.py b/ix-dev/community/portainer/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/portainer/templates/library/base_v2_0_30/volumes.py rename to ix-dev/community/portainer/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/postgres/app.yaml b/ix-dev/community/postgres/app.yaml index 1975105631..40ef50fe6b 100644 --- a/ix-dev/community/postgres/app.yaml +++ b/ix-dev/community/postgres/app.yaml @@ -9,8 +9,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/postgres/icons/icon.png keywords: - database -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -27,4 +27,4 @@ sources: - https://hub.docker.com/_/postgres title: Postgres train: community -version: 1.0.8 +version: 1.0.9 diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/container.py b/ix-dev/community/postgres/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/postgres/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/deps.py b/ix-dev/community/postgres/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/postgres/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/devices.py b/ix-dev/community/postgres/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/postgres/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/environment.py b/ix-dev/community/postgres/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/postgres/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/functions.py b/ix-dev/community/postgres/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/postgres/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/postgres/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/postgres/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/storage.py b/ix-dev/community/postgres/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/postgres/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/validations.py b/ix-dev/community/postgres/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/postgres/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/configs.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/container.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/depends.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/deps.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/device.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/device.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/devices.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/dns.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/environment.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/error.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/error.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/functions.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/labels.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/notes.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/portal.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/portals.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/ports.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/render.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/render.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/resources.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/restart.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/storage.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/postgres/templates/library/base_v2_1_0/validations.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/postgres/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/postgres/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/postgres/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/postgres/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/postgres/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/prowlarr/app.yaml b/ix-dev/community/prowlarr/app.yaml index 1f17e099e8..37878a75cd 100644 --- a/ix-dev/community/prowlarr/app.yaml +++ b/ix-dev/community/prowlarr/app.yaml @@ -9,8 +9,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/prowlarr/icons/icon.png keywords: - indexer -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -29,4 +29,4 @@ sources: - https://prowlarr.com title: Prowlarr train: community -version: 1.3.3 +version: 1.3.4 diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/container.py b/ix-dev/community/prowlarr/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/deps.py b/ix-dev/community/prowlarr/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/devices.py b/ix-dev/community/prowlarr/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/environment.py b/ix-dev/community/prowlarr/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/functions.py b/ix-dev/community/prowlarr/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/prowlarr/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/storage.py b/ix-dev/community/prowlarr/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/validations.py b/ix-dev/community/prowlarr/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/configs.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/container.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/depends.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/deps.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/device.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/device.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/devices.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/dns.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/environment.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/error.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/error.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/functions.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/labels.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/notes.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/portal.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/portals.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/ports.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/render.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/render.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/resources.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/restart.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/storage.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_1_0/validations.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/prowlarr/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/prowlarr/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/prowlarr/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/prowlarr/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/qbittorrent/app.yaml b/ix-dev/community/qbittorrent/app.yaml index 64a1cc9dee..a3f0292dd5 100644 --- a/ix-dev/community/qbittorrent/app.yaml +++ b/ix-dev/community/qbittorrent/app.yaml @@ -11,8 +11,8 @@ keywords: - media - torrent - download -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -31,4 +31,4 @@ sources: - https://www.qbittorrent.org/ title: qBittorrent train: community -version: 1.1.8 +version: 1.1.9 diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/container.py b/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/deps.py b/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/devices.py b/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/environment.py b/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/functions.py b/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/storage.py b/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/validations.py b/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/configs.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/container.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/depends.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/deps.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/device.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/device.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/devices.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/dns.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/environment.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/error.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/error.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/functions.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/labels.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/notes.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/portal.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/portals.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/ports.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/render.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/render.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/resources.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/restart.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/storage.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/validations.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/qbittorrent/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/qbittorrent/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/qbittorrent/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/qbittorrent/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/radarr/app.yaml b/ix-dev/community/radarr/app.yaml index bd433eb966..7841df0848 100644 --- a/ix-dev/community/radarr/app.yaml +++ b/ix-dev/community/radarr/app.yaml @@ -9,8 +9,8 @@ icon: https://media.sys.truenas.net/apps/radarr/icons/icon.png keywords: - media - movies -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -32,4 +32,4 @@ sources: - https://github.com/Radarr/Radarr title: Radarr train: community -version: 1.2.2 +version: 1.2.3 diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/container.py b/ix-dev/community/radarr/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/radarr/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/deps.py b/ix-dev/community/radarr/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/radarr/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/devices.py b/ix-dev/community/radarr/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/radarr/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/environment.py b/ix-dev/community/radarr/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/radarr/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/functions.py b/ix-dev/community/radarr/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/radarr/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/radarr/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/radarr/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/storage.py b/ix-dev/community/radarr/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/radarr/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/validations.py b/ix-dev/community/radarr/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/radarr/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/configs.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/container.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/depends.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/deps.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/device.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/device.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/devices.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/dns.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/environment.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/error.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/error.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/functions.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/labels.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/notes.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/portal.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/portals.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/ports.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/render.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/render.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/resources.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/restart.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/storage.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/radarr/templates/library/base_v2_1_0/validations.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/radarr/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/radarr/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/radarr/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/radarr/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/radarr/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/readarr/app.yaml b/ix-dev/community/readarr/app.yaml index 4ea5361269..41a9d88cf6 100644 --- a/ix-dev/community/readarr/app.yaml +++ b/ix-dev/community/readarr/app.yaml @@ -11,8 +11,8 @@ keywords: - media - ebook - audiobook -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -33,4 +33,4 @@ sources: - https://github.com/Readarr/Readarr title: Readarr train: community -version: 1.1.2 +version: 1.1.3 diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/container.py b/ix-dev/community/readarr/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/readarr/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/deps.py b/ix-dev/community/readarr/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/readarr/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/devices.py b/ix-dev/community/readarr/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/readarr/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/environment.py b/ix-dev/community/readarr/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/readarr/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/functions.py b/ix-dev/community/readarr/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/readarr/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/readarr/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/readarr/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/storage.py b/ix-dev/community/readarr/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/readarr/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/validations.py b/ix-dev/community/readarr/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/readarr/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/configs.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/container.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/depends.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/deps.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/device.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/device.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/devices.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/dns.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/environment.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/error.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/error.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/functions.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/labels.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/notes.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/portal.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/portals.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/ports.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/render.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/render.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/resources.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/restart.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/storage.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/readarr/templates/library/base_v2_1_0/validations.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/readarr/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/readarr/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/readarr/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/readarr/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/readarr/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/recyclarr/app.yaml b/ix-dev/community/recyclarr/app.yaml index 2136c2254c..883942ca15 100644 --- a/ix-dev/community/recyclarr/app.yaml +++ b/ix-dev/community/recyclarr/app.yaml @@ -11,8 +11,8 @@ keywords: - sync - sonarr - radarr -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -30,4 +30,4 @@ sources: - https://github.com/recyclarr/recyclarr/tree/recyclarr title: Recyclarr train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/container.py b/ix-dev/community/recyclarr/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/deps.py b/ix-dev/community/recyclarr/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/devices.py b/ix-dev/community/recyclarr/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/environment.py b/ix-dev/community/recyclarr/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/functions.py b/ix-dev/community/recyclarr/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/recyclarr/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/validations.py b/ix-dev/community/recyclarr/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/configs.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/container.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/depends.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/deps.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/device.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/device.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/devices.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/dns.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/environment.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/error.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/error.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/functions.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/labels.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/notes.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/portal.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/portals.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/ports.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/render.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/render.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/resources.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/restart.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/storage.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_1_0/validations.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/recyclarr/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/recyclarr/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/recyclarr/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/recyclarr/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/redis/app.yaml b/ix-dev/community/redis/app.yaml index 1eb4f65733..60c682f1ed 100644 --- a/ix-dev/community/redis/app.yaml +++ b/ix-dev/community/redis/app.yaml @@ -9,8 +9,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/redis/icons/icon.png keywords: - cache -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -29,4 +29,4 @@ sources: - https://redis.io/ title: Redis train: community -version: 1.1.1 +version: 1.1.2 diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/container.py b/ix-dev/community/redis/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/redis/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/deps.py b/ix-dev/community/redis/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/redis/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/devices.py b/ix-dev/community/redis/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/redis/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/environment.py b/ix-dev/community/redis/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/redis/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/functions.py b/ix-dev/community/redis/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/redis/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/redis/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/redis/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/storage.py b/ix-dev/community/redis/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/redis/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/validations.py b/ix-dev/community/redis/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/redis/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/redis/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/configs.py b/ix-dev/community/redis/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/container.py b/ix-dev/community/redis/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/depends.py b/ix-dev/community/redis/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/redis/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/deps.py b/ix-dev/community/redis/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/redis/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/redis/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/redis/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/redis/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/device.py b/ix-dev/community/redis/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/device.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/devices.py b/ix-dev/community/redis/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/dns.py b/ix-dev/community/redis/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/environment.py b/ix-dev/community/redis/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/error.py b/ix-dev/community/redis/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/error.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/redis/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/functions.py b/ix-dev/community/redis/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/redis/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/labels.py b/ix-dev/community/redis/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/notes.py b/ix-dev/community/redis/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/portal.py b/ix-dev/community/redis/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/portals.py b/ix-dev/community/redis/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/ports.py b/ix-dev/community/redis/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/render.py b/ix-dev/community/redis/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/render.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/resources.py b/ix-dev/community/redis/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/restart.py b/ix-dev/community/redis/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/storage.py b/ix-dev/community/redis/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/redis/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/redis/templates/library/base_v2_1_0/validations.py b/ix-dev/community/redis/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/redis/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/redis/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/redis/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/redis/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/redis/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/redis/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/redis/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/redis/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/redis/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/roundcube/app.yaml b/ix-dev/community/roundcube/app.yaml index 801da98f2d..e4832d7da5 100644 --- a/ix-dev/community/roundcube/app.yaml +++ b/ix-dev/community/roundcube/app.yaml @@ -20,8 +20,8 @@ icon: https://media.sys.truenas.net/apps/roundcube/icons/icon.png keywords: - webmail - email -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -49,4 +49,4 @@ sources: - https://hub.docker.com/r/roundcube/roundcubemail/ title: Roundcube train: community -version: 1.1.3 +version: 1.1.4 diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/container.py b/ix-dev/community/roundcube/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/roundcube/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/deps.py b/ix-dev/community/roundcube/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/roundcube/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/devices.py b/ix-dev/community/roundcube/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/roundcube/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/environment.py b/ix-dev/community/roundcube/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/roundcube/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/functions.py b/ix-dev/community/roundcube/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/roundcube/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/roundcube/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/roundcube/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/storage.py b/ix-dev/community/roundcube/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/roundcube/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/validations.py b/ix-dev/community/roundcube/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/roundcube/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/configs.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/container.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/depends.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/deps.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/device.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/device.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/devices.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/dns.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/environment.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/error.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/error.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/functions.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/labels.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/notes.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/portal.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/portals.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/ports.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/render.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/render.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/resources.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/restart.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/storage.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/roundcube/templates/library/base_v2_1_0/validations.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/roundcube/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/roundcube/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/roundcube/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/roundcube/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/roundcube/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/rsyncd/app.yaml b/ix-dev/community/rsyncd/app.yaml index d3070002fd..39b38abe9b 100644 --- a/ix-dev/community/rsyncd/app.yaml +++ b/ix-dev/community/rsyncd/app.yaml @@ -22,8 +22,8 @@ keywords: - sync - rsync - file transfer -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -41,4 +41,4 @@ sources: - https://hub.docker.com/r/ixsystems/rsyncd title: Rsync Daemon train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/container.py b/ix-dev/community/rsyncd/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/deps.py b/ix-dev/community/rsyncd/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/devices.py b/ix-dev/community/rsyncd/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/environment.py b/ix-dev/community/rsyncd/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/functions.py b/ix-dev/community/rsyncd/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/rsyncd/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/validations.py b/ix-dev/community/rsyncd/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/configs.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/container.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/depends.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/deps.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/device.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/device.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/devices.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/dns.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/environment.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/error.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/error.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/functions.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/labels.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/notes.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/portal.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/portals.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/ports.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/render.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/render.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/resources.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/restart.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/storage.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_1_0/validations.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/rsyncd/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/rsyncd/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/rsyncd/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/rsyncd/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/rust-desk/app.yaml b/ix-dev/community/rust-desk/app.yaml index 5ffd50fa76..370b435692 100644 --- a/ix-dev/community/rust-desk/app.yaml +++ b/ix-dev/community/rust-desk/app.yaml @@ -9,8 +9,8 @@ icon: https://media.sys.truenas.net/apps/rust-desk/icons/icon.png keywords: - remote - desktop -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -28,4 +28,4 @@ sources: - https://github.com/rustdesk/rustdesk-server title: Rust Desk train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/container.py b/ix-dev/community/rust-desk/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/deps.py b/ix-dev/community/rust-desk/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/devices.py b/ix-dev/community/rust-desk/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/environment.py b/ix-dev/community/rust-desk/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/functions.py b/ix-dev/community/rust-desk/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/rust-desk/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/validations.py b/ix-dev/community/rust-desk/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/configs.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/container.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/depends.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/deps.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/device.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/device.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/devices.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/dns.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/environment.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/error.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/error.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/functions.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/labels.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/notes.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/portal.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/portals.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/ports.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/render.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/render.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/resources.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/restart.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/storage.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_1_0/validations.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/rust-desk/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/rust-desk/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/rust-desk/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/rust-desk/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/sabnzbd/app.yaml b/ix-dev/community/sabnzbd/app.yaml index 31c36d841f..d90ba010b5 100644 --- a/ix-dev/community/sabnzbd/app.yaml +++ b/ix-dev/community/sabnzbd/app.yaml @@ -10,8 +10,8 @@ keywords: - media - usenet - newsreader -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -32,4 +32,4 @@ sources: - https://sabnzbd.org/ title: SABnzbd train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/container.py b/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/deps.py b/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/devices.py b/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/environment.py b/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/functions.py b/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/validations.py b/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/configs.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/container.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/depends.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/deps.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/device.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/device.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/devices.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/dns.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/environment.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/error.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/error.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/functions.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/labels.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/notes.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/portal.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/portals.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/ports.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/render.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/render.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/resources.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/restart.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/storage.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/validations.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/sabnzbd/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/sabnzbd/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/sabnzbd/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/sabnzbd/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/scrutiny/app.yaml b/ix-dev/community/scrutiny/app.yaml index 27c679cf7e..8fd23ca407 100644 --- a/ix-dev/community/scrutiny/app.yaml +++ b/ix-dev/community/scrutiny/app.yaml @@ -18,8 +18,8 @@ icon: https://media.sys.truenas.net/apps/scrutiny/icons/icon.svg keywords: - disk - monitoring -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -37,4 +37,4 @@ sources: - https://github.com/AnalogJ/scrutiny title: Scrutiny train: community -version: 1.0.5 +version: 1.0.6 diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/container.py b/ix-dev/community/scrutiny/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/deps.py b/ix-dev/community/scrutiny/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/devices.py b/ix-dev/community/scrutiny/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/environment.py b/ix-dev/community/scrutiny/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/functions.py b/ix-dev/community/scrutiny/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/scrutiny/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/storage.py b/ix-dev/community/scrutiny/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/validations.py b/ix-dev/community/scrutiny/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/configs.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/container.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/depends.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/deps.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/device.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/device.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/devices.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/dns.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/environment.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/error.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/error.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/functions.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/labels.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/notes.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/portal.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/portals.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/ports.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/render.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/render.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/resources.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/restart.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/storage.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_1_0/validations.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/scrutiny/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/scrutiny/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/scrutiny/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/scrutiny/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/searxng/app.yaml b/ix-dev/community/searxng/app.yaml index 4d438064b9..18252bea76 100644 --- a/ix-dev/community/searxng/app.yaml +++ b/ix-dev/community/searxng/app.yaml @@ -12,8 +12,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/searxng/icons/icon.svg keywords: - search -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -31,4 +31,4 @@ sources: - https://github.com/searxng/searxng title: SearXNG train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/container.py b/ix-dev/community/searxng/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/searxng/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/deps.py b/ix-dev/community/searxng/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/searxng/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/devices.py b/ix-dev/community/searxng/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/searxng/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/environment.py b/ix-dev/community/searxng/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/searxng/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/functions.py b/ix-dev/community/searxng/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/searxng/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/searxng/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/searxng/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/validations.py b/ix-dev/community/searxng/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/searxng/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/configs.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/container.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/depends.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/deps.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/device.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/device.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/devices.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/dns.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/environment.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/error.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/error.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/functions.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/labels.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/notes.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/portal.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/portals.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/ports.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/render.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/render.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/resources.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/restart.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/storage.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/searxng/templates/library/base_v2_1_0/validations.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/searxng/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/searxng/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/searxng/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/searxng/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/searxng/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/sftpgo/app.yaml b/ix-dev/community/sftpgo/app.yaml index 77fbeee8d7..b1e2e027ec 100644 --- a/ix-dev/community/sftpgo/app.yaml +++ b/ix-dev/community/sftpgo/app.yaml @@ -9,8 +9,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/sftpgo/icons/icon.png keywords: - sftp -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -28,4 +28,4 @@ sources: - https://github.com/drakkan/sftpgo title: SFTPGo train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/container.py b/ix-dev/community/sftpgo/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/deps.py b/ix-dev/community/sftpgo/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/devices.py b/ix-dev/community/sftpgo/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/environment.py b/ix-dev/community/sftpgo/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/functions.py b/ix-dev/community/sftpgo/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/sftpgo/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/validations.py b/ix-dev/community/sftpgo/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/configs.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/container.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/depends.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/deps.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/device.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/device.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/devices.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/dns.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/environment.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/error.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/error.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/functions.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/labels.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/notes.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/portal.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/portals.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/ports.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/render.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/render.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/resources.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/restart.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/storage.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_1_0/validations.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/sftpgo/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/sftpgo/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/sftpgo/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/sftpgo/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/sonarr/app.yaml b/ix-dev/community/sonarr/app.yaml index 7f3a456fb7..67db436265 100644 --- a/ix-dev/community/sonarr/app.yaml +++ b/ix-dev/community/sonarr/app.yaml @@ -9,8 +9,8 @@ icon: https://media.sys.truenas.net/apps/sonarr/icons/icon.png keywords: - media - series -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -31,4 +31,4 @@ sources: - https://github.com/Sonarr/Sonarr title: Sonarr train: community -version: 1.1.2 +version: 1.1.3 diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/container.py b/ix-dev/community/sonarr/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/sonarr/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/deps.py b/ix-dev/community/sonarr/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/sonarr/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/devices.py b/ix-dev/community/sonarr/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/sonarr/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/environment.py b/ix-dev/community/sonarr/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/sonarr/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/functions.py b/ix-dev/community/sonarr/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/sonarr/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/sonarr/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/sonarr/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/storage.py b/ix-dev/community/sonarr/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/sonarr/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/validations.py b/ix-dev/community/sonarr/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/sonarr/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/configs.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/container.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/depends.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/deps.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/device.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/device.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/devices.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/dns.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/environment.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/error.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/error.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/functions.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/labels.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/notes.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/portal.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/portals.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/ports.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/render.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/render.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/resources.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/restart.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/storage.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/sonarr/templates/library/base_v2_1_0/validations.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/sonarr/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/sonarr/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/sonarr/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/sonarr/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/sonarr/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/tailscale/app.yaml b/ix-dev/community/tailscale/app.yaml index 57ce609da0..ec748807ff 100644 --- a/ix-dev/community/tailscale/app.yaml +++ b/ix-dev/community/tailscale/app.yaml @@ -23,8 +23,8 @@ icon: https://media.sys.truenas.net/apps/tailscale/icons/icon.png keywords: - vpn - tailscale -lib_version: 2.0.24 -lib_version_hash: 283cc9c5d0a45474968e1280324fab8fc7e176c41fdafdb6dcf9b9a74efebb9c +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -42,4 +42,4 @@ sources: - https://hub.docker.com/r/tailscale/tailscale title: Tailscale train: community -version: 1.2.0 +version: 1.2.1 diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/container.py b/ix-dev/community/tailscale/templates/library/base_v2_0_24/container.py deleted file mode 100644 index e49370c28b..0000000000 --- a/ix-dev/community/tailscale/templates/library/base_v2_0_24/container.py +++ /dev/null @@ -1,307 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/deps.py b/ix-dev/community/tailscale/templates/library/base_v2_0_24/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/tailscale/templates/library/base_v2_0_24/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/devices.py b/ix-dev/community/tailscale/templates/library/base_v2_0_24/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/tailscale/templates/library/base_v2_0_24/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/environment.py b/ix-dev/community/tailscale/templates/library/base_v2_0_24/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/tailscale/templates/library/base_v2_0_24/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/functions.py b/ix-dev/community/tailscale/templates/library/base_v2_0_24/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/tailscale/templates/library/base_v2_0_24/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/healthcheck.py b/ix-dev/community/tailscale/templates/library/base_v2_0_24/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/tailscale/templates/library/base_v2_0_24/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_container.py b/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_container.py deleted file mode 100644 index 84d712a403..0000000000 --- a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_container.py +++ /dev/null @@ -1,314 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_deps.py b/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_device.py b/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_environment.py b/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_functions.py b/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_healthcheck.py b/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/validations.py b/ix-dev/community/tailscale/templates/library/base_v2_0_24/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/tailscale/templates/library/base_v2_0_24/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/__init__.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/__init__.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/configs.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/configs.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/container.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/depends.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/depends.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/deploy.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/deploy.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/deps.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/device.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/device.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/devices.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/dns.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/dns.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/environment.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/error.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/error.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/formatter.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/formatter.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/functions.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/labels.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/labels.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/notes.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/notes.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/portal.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/portal.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/portals.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/portals.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/ports.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/ports.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/render.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/render.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/resources.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/resources.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/restart.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/restart.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/storage.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/__init__.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/__init__.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_build_image.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_build_image.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_configs.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_configs.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_depends.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_depends.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_dns.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_dns.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_formatter.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_formatter.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_labels.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_labels.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_notes.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_notes.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_portal.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_portal.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_ports.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_ports.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_render.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_render.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_resources.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_resources.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_restart.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/tests/test_restart.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/tailscale/templates/library/base_v2_1_0/validations.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/tailscale/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/volume_mount.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/volume_mount.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/volume_mount_types.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/volume_mount_types.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/volume_sources.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/volume_sources.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/volume_types.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/volume_types.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/tailscale/templates/library/base_v2_0_24/volumes.py b/ix-dev/community/tailscale/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/tailscale/templates/library/base_v2_0_24/volumes.py rename to ix-dev/community/tailscale/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/tautulli/app.yaml b/ix-dev/community/tautulli/app.yaml index 7dd957e59e..e727bcc8c9 100644 --- a/ix-dev/community/tautulli/app.yaml +++ b/ix-dev/community/tautulli/app.yaml @@ -11,8 +11,8 @@ keywords: - media - analytics - notifications -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -34,4 +34,4 @@ sources: - https://github.com/Tautulli/Tautulli title: Tautulli train: community -version: 1.1.2 +version: 1.1.3 diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/container.py b/ix-dev/community/tautulli/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/tautulli/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/deps.py b/ix-dev/community/tautulli/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/tautulli/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/devices.py b/ix-dev/community/tautulli/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/tautulli/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/environment.py b/ix-dev/community/tautulli/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/tautulli/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/functions.py b/ix-dev/community/tautulli/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/tautulli/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/tautulli/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/tautulli/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/storage.py b/ix-dev/community/tautulli/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/tautulli/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/validations.py b/ix-dev/community/tautulli/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/tautulli/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/configs.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/container.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/depends.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/deps.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/device.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/device.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/devices.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/dns.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/environment.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/error.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/error.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/functions.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/labels.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/notes.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/portal.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/portals.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/ports.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/render.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/render.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/resources.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/restart.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/storage.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/tautulli/templates/library/base_v2_1_0/validations.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/tautulli/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/tautulli/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/tautulli/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/tautulli/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/tautulli/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/tdarr/app.yaml b/ix-dev/community/tdarr/app.yaml index ef8fa3c3cf..0c537b2771 100644 --- a/ix-dev/community/tdarr/app.yaml +++ b/ix-dev/community/tdarr/app.yaml @@ -18,8 +18,8 @@ icon: https://media.sys.truenas.net/apps/tdarr/icons/icon.png keywords: - encode - transcode -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -40,4 +40,4 @@ sources: - https://docs.tdarr.io/docs title: Tdarr train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/container.py b/ix-dev/community/tdarr/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/tdarr/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/deps.py b/ix-dev/community/tdarr/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/tdarr/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/devices.py b/ix-dev/community/tdarr/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/tdarr/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/environment.py b/ix-dev/community/tdarr/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/tdarr/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/functions.py b/ix-dev/community/tdarr/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/tdarr/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/tdarr/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/tdarr/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/validations.py b/ix-dev/community/tdarr/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/tdarr/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/configs.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/container.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/depends.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/deps.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/device.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/device.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/devices.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/dns.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/environment.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/error.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/error.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/functions.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/labels.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/notes.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/portal.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/portals.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/ports.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/render.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/render.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/resources.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/restart.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/storage.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/tdarr/templates/library/base_v2_1_0/validations.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/tdarr/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/tdarr/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/tdarr/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/tdarr/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/tdarr/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/terraria/app.yaml b/ix-dev/community/terraria/app.yaml index 2bde26b399..6ff7489fa3 100644 --- a/ix-dev/community/terraria/app.yaml +++ b/ix-dev/community/terraria/app.yaml @@ -13,8 +13,8 @@ keywords: - game - terraria - world -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -32,4 +32,4 @@ sources: - https://github.com/ryansheehan/terraria title: Terraria train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/container.py b/ix-dev/community/terraria/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/terraria/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/deps.py b/ix-dev/community/terraria/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/terraria/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/devices.py b/ix-dev/community/terraria/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/terraria/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/environment.py b/ix-dev/community/terraria/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/terraria/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/functions.py b/ix-dev/community/terraria/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/terraria/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/terraria/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/terraria/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/validations.py b/ix-dev/community/terraria/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/terraria/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/configs.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/container.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/depends.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/deps.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/device.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/device.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/devices.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/dns.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/environment.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/error.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/error.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/functions.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/labels.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/notes.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/portal.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/portals.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/ports.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/render.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/render.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/resources.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/restart.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/storage.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/terraria/templates/library/base_v2_1_0/validations.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/terraria/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/terraria/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/terraria/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/terraria/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/terraria/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/tftpd-hpa/app.yaml b/ix-dev/community/tftpd-hpa/app.yaml index 0b3abad6c2..5c644b4771 100644 --- a/ix-dev/community/tftpd-hpa/app.yaml +++ b/ix-dev/community/tftpd-hpa/app.yaml @@ -17,8 +17,8 @@ icon: https://media.sys.truenas.net/apps/tftpd-hpa/icons/icon.png keywords: - tftp - netboot -lib_version: 2.0.32 -lib_version_hash: 4a0bf69cccda322e191eab36ab81ca6d0c8e5d64a0b2fa117c609804b55b86c6 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -36,4 +36,4 @@ sources: - https://hub.docker.com/r/ixsystems/tftpd-hpa title: TFTP Server train: community -version: 1.1.1 +version: 1.1.2 diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/container.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/deps.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/devices.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/environment.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/healthcheck.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_container.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_deps.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_device.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_environment.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_healthcheck.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/validations.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/__init__.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/__init__.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/configs.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/configs.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/container.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/depends.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/depends.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/deploy.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/deploy.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/deps.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/device.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/device.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/devices.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/dns.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/dns.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/environment.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/error.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/error.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/formatter.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/formatter.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/functions.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/labels.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/labels.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/notes.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/notes.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/portal.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/portal.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/portals.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/portals.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/ports.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/ports.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/render.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/render.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/resources.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/resources.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/restart.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/restart.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/storage.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/__init__.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/__init__.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_build_image.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_build_image.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_configs.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_configs.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_depends.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_depends.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_dns.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_dns.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_formatter.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_formatter.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_labels.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_labels.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_notes.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_notes.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_portal.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_portal.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_ports.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_ports.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_render.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_render.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_resources.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_resources.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_restart.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/tests/test_restart.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/validations.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/volume_mount.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/volume_mount.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/volume_mount_types.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/volume_mount_types.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/volume_sources.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/volume_sources.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/volume_types.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/volume_types.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/volumes.py b/ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/tftpd-hpa/templates/library/base_v2_0_32/volumes.py rename to ix-dev/community/tftpd-hpa/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/tiny-media-manager/app.yaml b/ix-dev/community/tiny-media-manager/app.yaml index 1e8eca3007..2ed74d8c9f 100644 --- a/ix-dev/community/tiny-media-manager/app.yaml +++ b/ix-dev/community/tiny-media-manager/app.yaml @@ -16,8 +16,8 @@ keywords: - media - tv-shows - movies -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -39,4 +39,4 @@ sources: - https://hub.docker.com/r/tinymediamanager/tinymediamanager title: Tiny Media Manager train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/container.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/deps.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/devices.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/environment.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/functions.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/validations.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/configs.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/container.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/depends.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/deps.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/device.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/device.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/devices.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/dns.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/environment.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/error.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/error.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/functions.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/labels.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/notes.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/portal.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/portals.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/ports.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/render.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/render.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/resources.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/restart.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/storage.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/validations.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/tiny-media-manager/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/tiny-media-manager/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/transmission/app.yaml b/ix-dev/community/transmission/app.yaml index d9c598429a..a31afd9125 100644 --- a/ix-dev/community/transmission/app.yaml +++ b/ix-dev/community/transmission/app.yaml @@ -10,8 +10,8 @@ keywords: - media - torrent - download -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -29,4 +29,4 @@ sources: - https://transmissionbt.com/ title: Transmission train: community -version: 1.1.1 +version: 1.1.2 diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/container.py b/ix-dev/community/transmission/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/transmission/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/deps.py b/ix-dev/community/transmission/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/transmission/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/devices.py b/ix-dev/community/transmission/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/transmission/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/environment.py b/ix-dev/community/transmission/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/transmission/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/functions.py b/ix-dev/community/transmission/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/transmission/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/transmission/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/transmission/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/storage.py b/ix-dev/community/transmission/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/transmission/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/validations.py b/ix-dev/community/transmission/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/transmission/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/configs.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/container.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/depends.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/deps.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/device.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/device.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/devices.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/dns.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/environment.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/error.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/error.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/functions.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/labels.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/notes.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/portal.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/portals.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/ports.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/render.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/render.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/resources.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/restart.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/storage.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/transmission/templates/library/base_v2_1_0/validations.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/transmission/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/transmission/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/transmission/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/transmission/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/transmission/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/twofactor-auth/app.yaml b/ix-dev/community/twofactor-auth/app.yaml index 85088d1820..63a6ad4919 100644 --- a/ix-dev/community/twofactor-auth/app.yaml +++ b/ix-dev/community/twofactor-auth/app.yaml @@ -11,8 +11,8 @@ keywords: - security - 2fa - otp -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -31,4 +31,4 @@ sources: - https://hub.docker.com/r/2fauth/2fauth/ title: 2FAuth train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/container.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/deps.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/devices.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/environment.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/functions.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/validations.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/configs.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/container.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/depends.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/deps.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/device.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/device.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/devices.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/dns.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/environment.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/error.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/error.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/functions.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/labels.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/notes.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/portal.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/portals.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/ports.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/render.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/render.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/resources.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/restart.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/storage.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/validations.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/twofactor-auth/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/twofactor-auth/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/unifi-controller/app.yaml b/ix-dev/community/unifi-controller/app.yaml index b14dfad895..fc52a60d1c 100644 --- a/ix-dev/community/unifi-controller/app.yaml +++ b/ix-dev/community/unifi-controller/app.yaml @@ -10,8 +10,8 @@ keywords: - controller - unifi - network -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -30,4 +30,4 @@ sources: - https://hub.docker.com/r/goofball222/unifi title: Unifi Controller train: community -version: 1.3.0 +version: 1.3.1 diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/container.py b/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/deps.py b/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/devices.py b/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/environment.py b/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/functions.py b/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/validations.py b/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/configs.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/container.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/depends.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/deps.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/device.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/device.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/devices.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/dns.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/environment.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/error.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/error.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/functions.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/labels.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/notes.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/portal.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/portals.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/ports.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/render.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/render.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/resources.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/restart.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/storage.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/validations.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/unifi-controller/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/unifi-controller/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/unifi-controller/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/unifi-controller/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/unifi-protect-backup/app.yaml b/ix-dev/community/unifi-protect-backup/app.yaml index 8ebb84cfa4..105e7e1fee 100644 --- a/ix-dev/community/unifi-protect-backup/app.yaml +++ b/ix-dev/community/unifi-protect-backup/app.yaml @@ -18,8 +18,8 @@ icon: https://media.sys.truenas.net/apps/unifi-protect-backup/icons/icon.svg keywords: - backup - unifi-protect -lib_version: 2.0.29 -lib_version_hash: 3c2dd83143491fdfc23e5e13de9db0aa79bb7846a2b7c32a76a479e334c54590 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -37,4 +37,4 @@ sources: - https://github.com/ep1cman/unifi-protect-backup/pkgs/container/unifi-protect-backup title: Unifi Protect Backup train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/container.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/deps.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/devices.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/environment.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/functions.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/healthcheck.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_container.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_deps.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_device.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_environment.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_functions.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_healthcheck.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/validations.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/__init__.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/__init__.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/configs.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/configs.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/container.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/depends.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/depends.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/deploy.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/deploy.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/deps.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/device.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/device.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/devices.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/dns.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/dns.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/environment.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/error.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/error.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/formatter.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/formatter.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/functions.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/labels.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/labels.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/notes.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/notes.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/portal.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/portal.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/portals.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/portals.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/ports.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/ports.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/render.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/render.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/resources.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/resources.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/restart.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/restart.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/storage.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/__init__.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/__init__.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_build_image.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_build_image.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_configs.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_configs.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_depends.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_depends.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_dns.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_dns.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_formatter.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_formatter.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_labels.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_labels.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_notes.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_notes.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_portal.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_portal.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_ports.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_ports.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_render.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_render.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_resources.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_resources.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_restart.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/tests/test_restart.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/validations.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/volume_mount.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/volume_mount.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/volume_mount_types.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/volume_mount_types.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/volume_sources.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/volume_sources.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/volume_types.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/volume_types.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/volumes.py b/ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/unifi-protect-backup/templates/library/base_v2_0_29/volumes.py rename to ix-dev/community/unifi-protect-backup/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/uptime-kuma/app.yaml b/ix-dev/community/uptime-kuma/app.yaml index ec28f33fc0..eb3ce0ffb2 100644 --- a/ix-dev/community/uptime-kuma/app.yaml +++ b/ix-dev/community/uptime-kuma/app.yaml @@ -11,8 +11,8 @@ icon: https://media.sys.truenas.net/apps/uptime-kuma/icons/icon.svg keywords: - uptime - monitor -lib_version: 2.0.23 -lib_version_hash: 8909de0f230ddb07f219f65eb8e67f9b8a112e9c310bee801834248854aa91af +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -31,4 +31,4 @@ sources: - https://github.com/louislam/uptime-kuma title: Uptime Kuma train: community -version: 1.0.7 +version: 1.0.8 diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/container.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/container.py deleted file mode 100644 index 34df9ec83b..0000000000 --- a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/container.py +++ /dev/null @@ -1,304 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/deps.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/devices.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/environment.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/functions.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/healthcheck.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/storage.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/storage.py deleted file mode 100644 index bf39d45daa..0000000000 --- a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def _add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_container.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_container.py deleted file mode 100644 index 5e51b86198..0000000000 --- a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_container.py +++ /dev/null @@ -1,294 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_deps.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_device.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_environment.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_functions.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_healthcheck.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_volumes.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_volumes.py deleted file mode 100644 index 2084de09e0..0000000000 --- a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage._add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage._add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage._add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/validations.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/__init__.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/__init__.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/configs.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/configs.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/container.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/depends.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/depends.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/deploy.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/deploy.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/deps.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/device.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/device.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/devices.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/dns.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/dns.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/environment.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/error.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/error.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/formatter.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/formatter.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/functions.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/labels.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/labels.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/notes.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/notes.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/portal.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/portal.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/portals.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/portals.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/ports.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/ports.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/render.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/render.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/resources.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/resources.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/restart.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/restart.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/storage.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/__init__.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/__init__.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_build_image.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_build_image.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_configs.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_configs.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_depends.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_depends.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_dns.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_dns.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_formatter.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_formatter.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_labels.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_labels.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_notes.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_notes.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_portal.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_portal.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_ports.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_ports.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_render.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_render.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_resources.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_resources.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_restart.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/tests/test_restart.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/validations.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/volume_mount.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/volume_mount.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/volume_mount_types.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/volume_mount_types.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/volume_sources.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/volume_sources.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/volume_types.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/volume_types.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/volumes.py b/ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/uptime-kuma/templates/library/base_v2_0_23/volumes.py rename to ix-dev/community/uptime-kuma/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/vaultwarden/app.yaml b/ix-dev/community/vaultwarden/app.yaml index 428153caef..a6ce1c3ff7 100644 --- a/ix-dev/community/vaultwarden/app.yaml +++ b/ix-dev/community/vaultwarden/app.yaml @@ -10,8 +10,8 @@ icon: https://media.sys.truenas.net/apps/vaultwarden/icons/icon.png keywords: - password - manager -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -34,4 +34,4 @@ sources: - https://github.com/dani-garcia/vaultwarden title: Vaultwarden train: community -version: 1.1.11 +version: 1.1.12 diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/container.py b/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/deps.py b/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/devices.py b/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/environment.py b/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/functions.py b/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/storage.py b/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/validations.py b/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/configs.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/container.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/depends.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/deps.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/device.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/device.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/devices.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/dns.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/environment.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/error.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/error.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/functions.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/labels.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/notes.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/portal.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/portals.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/ports.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/render.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/render.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/resources.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/restart.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/storage.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/validations.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/vaultwarden/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/vaultwarden/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/vaultwarden/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/vaultwarden/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/vikunja/app.yaml b/ix-dev/community/vikunja/app.yaml index 4d6ccf655b..c45d736fa0 100644 --- a/ix-dev/community/vikunja/app.yaml +++ b/ix-dev/community/vikunja/app.yaml @@ -8,8 +8,8 @@ host_mounts: [] icon: https://media.sys.truenas.net/apps/vikunja/icons/icon.png keywords: - todo -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -46,4 +46,4 @@ sources: - https://vikunja.io/ title: Vikunja train: community -version: 1.3.5 +version: 1.3.6 diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/container.py b/ix-dev/community/vikunja/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/vikunja/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/deps.py b/ix-dev/community/vikunja/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/vikunja/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/devices.py b/ix-dev/community/vikunja/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/vikunja/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/environment.py b/ix-dev/community/vikunja/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/vikunja/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/functions.py b/ix-dev/community/vikunja/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/vikunja/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/vikunja/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/vikunja/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/storage.py b/ix-dev/community/vikunja/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/vikunja/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/validations.py b/ix-dev/community/vikunja/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/vikunja/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/configs.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/container.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/depends.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/deps.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/device.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/device.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/devices.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/dns.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/environment.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/error.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/error.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/functions.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/labels.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/notes.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/portal.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/portals.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/ports.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/render.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/render.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/resources.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/restart.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/storage.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/vikunja/templates/library/base_v2_1_0/validations.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/vikunja/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/vikunja/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/vikunja/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/vikunja/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/vikunja/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/webdav/app.yaml b/ix-dev/community/webdav/app.yaml index 21bb77b50e..3064d79bb7 100644 --- a/ix-dev/community/webdav/app.yaml +++ b/ix-dev/community/webdav/app.yaml @@ -10,8 +10,8 @@ icon: https://media.sys.truenas.net/apps/webdav/icons/icon.png keywords: - webdav - file-sharing -lib_version: 2.0.21 -lib_version_hash: e2faccd282b768e411919a7386a03e8491d1a7fda2da586dcf9af0d412733b8a +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -28,4 +28,4 @@ sources: - http://www.webdav.org/ title: WebDAV train: community -version: 1.1.1 +version: 1.1.2 diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/container.py b/ix-dev/community/webdav/templates/library/base_v2_0_21/container.py deleted file mode 100644 index 1994caf704..0000000000 --- a/ix-dev/community/webdav/templates/library/base_v2_0_21/container.py +++ /dev/null @@ -1,300 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/deps.py b/ix-dev/community/webdav/templates/library/base_v2_0_21/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/webdav/templates/library/base_v2_0_21/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/devices.py b/ix-dev/community/webdav/templates/library/base_v2_0_21/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/webdav/templates/library/base_v2_0_21/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/environment.py b/ix-dev/community/webdav/templates/library/base_v2_0_21/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/webdav/templates/library/base_v2_0_21/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/functions.py b/ix-dev/community/webdav/templates/library/base_v2_0_21/functions.py deleted file mode 100644 index a8799c3963..0000000000 --- a/ix-dev/community/webdav/templates/library/base_v2_0_21/functions.py +++ /dev/null @@ -1,111 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError -except ImportError: - from error import RenderError - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string, chars, key): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - } diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/healthcheck.py b/ix-dev/community/webdav/templates/library/base_v2_0_21/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/webdav/templates/library/base_v2_0_21/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/storage.py b/ix-dev/community/webdav/templates/library/base_v2_0_21/storage.py deleted file mode 100644 index 3e5b629790..0000000000 --- a/ix-dev/community/webdav/templates/library/base_v2_0_21/storage.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .validations import valid_fs_path_or_raise - from .volume_mount import VolumeMount -except ImportError: - from error import RenderError - from validations import valid_fs_path_or_raise - from volume_mount import VolumeMount - - -class IxStorageTmpfsConfig(TypedDict): - size: NotRequired[int] - mode: NotRequired[str] - - -class AclConfig(TypedDict, total=False): - path: str - - -class IxStorageHostPathConfig(TypedDict): - path: NotRequired[str] # Either this or acl.path must be set - acl_enable: NotRequired[bool] - acl: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageIxVolumeConfig(TypedDict): - dataset_name: str - acl_enable: NotRequired[bool] - acl_entries: NotRequired[AclConfig] - create_host_path: NotRequired[bool] - propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] - auto_permissions: NotRequired[bool] # Only when acl_enable is false - - -class IxStorageVolumeConfig(TypedDict): - volume_name: NotRequired[str] - nocopy: NotRequired[bool] - auto_permissions: NotRequired[bool] - - -class IxStorageNfsConfig(TypedDict): - server: str - path: str - options: NotRequired[list[str]] - - -class IxStorageCifsConfig(TypedDict): - server: str - path: str - username: str - password: str - domain: NotRequired[str] - options: NotRequired[list[str]] - - -IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] -IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] -IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] - - -class IxStorage(TypedDict): - type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] - read_only: NotRequired[bool] - - ix_volume_config: NotRequired[IxStorageIxVolumeConfig] - host_path_config: NotRequired[IxStorageHostPathConfig] - tmpfs_config: NotRequired[IxStorageTmpfsConfig] - volume_config: NotRequired[IxStorageVolumeConfig] - nfs_config: NotRequired[IxStorageNfsConfig] - cifs_config: NotRequired[IxStorageCifsConfig] - - -class Storage: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._volume_mounts: set[VolumeMount] = set() - - def add(self, mount_path: str, config: "IxStorage"): - mount_path = valid_fs_path_or_raise(mount_path) - if mount_path in [m.mount_path for m in self._volume_mounts]: - raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") - - volume_mount = VolumeMount(self._render_instance, mount_path, config) - self._volume_mounts.add(volume_mount) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - cfg: "IxStorage" = { - "type": "host_path", - "read_only": read_only, - "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, - } - self.add(mount_path, cfg) - - def has_mounts(self) -> bool: - return bool(self._volume_mounts) - - def render(self): - return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_container.py b/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_container.py deleted file mode 100644 index 35f26f43af..0000000000 --- a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_container.py +++ /dev/null @@ -1,273 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_deps.py b/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_device.py b/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_environment.py b/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_functions.py b/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_functions.py deleted file mode 100644 index a75e7c4084..0000000000 --- a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_functions.py +++ /dev/null @@ -1,65 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_healthcheck.py b/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_volumes.py b/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_volumes.py deleted file mode 100644 index e0ae9a6953..0000000000 --- a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_volumes.py +++ /dev/null @@ -1,666 +0,0 @@ -import pytest - - -from render import Render -from formatter import get_hashed_name_for_volume - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_volume_invalid_type(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "invalid_type"}) - - -def test_add_volume_empty_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_storage("", {"type": "tmpfs"}) - - -def test_add_volume_duplicate_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_storage("/some/path", {"type": "tmpfs"}) - with pytest.raises(Exception): - c1.add_storage("/some/path", {"type": "tmpfs"}) - - -def test_add_volume_host_path_invalid_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_host_path_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path"} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_with_acl_no_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", host_path_config) - - -def test_add_host_path_volume_mount(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_acl(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = { - "type": "host_path", - "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, - } - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test/acl", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_propagation(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "slave"}, - } - ] - - -def test_add_host_path_volume_mount_with_create_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rprivate"}, - } - ] - - -def test_add_host_path_volume_mount_with_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} - c1.add_storage("/some/path", host_path_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_invalid_dataset_name(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_no_ix_volume_config(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume"} - with pytest.raises(Exception): - c1.add_storage("/some/path", ix_volume_config) - - -def test_add_ix_volume_volume_mount(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_ix_volume_volume_mount_with_options(mock_values): - mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - ix_volume_config = { - "type": "ix_volume", - "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, - } - c1.add_storage("/some/path", ix_volume_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/mnt/test", - "target": "/some/path", - "read_only": False, - "bind": {"create_host_path": True, "propagation": "rslave"}, - } - ] - - -def test_cifs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_username(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_missing_password(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_without_cifs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = {"type": "cifs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["verbose=true", "verbose=true"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["user=username"], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": {"verbose": True}, - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_cifs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_config = { - "type": "cifs", - "cifs_config": { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": [{"verbose": True}], - }, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", cifs_config) - - -def test_add_cifs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_cifs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - cifs_inner_config = { - "server": "server", - "path": "/path", - "username": "user", - "password": "pas$word", - "options": ["vers=3.0", "verbose=true"], - } - cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} - c1.add_storage("/some/path", cifs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "cifs", - "device": "//server/path", - "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_missing_server(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_missing_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_without_nfs_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs"} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_duplicate_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = { - "type": "nfs", - "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, - } - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_disallowed_option(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_nfs_volume_invalid_options2(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} - with pytest.raises(Exception): - c1.add_storage("/some/path", nfs_config) - - -def test_add_nfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path"} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_nfs_volume_with_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} - nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} - c1.add_storage("/some/path", nfs_config) - output = render.render() - vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) - assert output["volumes"] == { - vol_name: { - "driver_opts": { - "type": "nfs", - "device": ":/path", - "o": "addr=server,verbose=true,vers=3.0", - } - } - } - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} - ] - - -def test_tmpfs_invalid_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_zero_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_invalid_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_tmpfs_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "tmpfs"} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "tmpfs", - "target": "/some/path", - "read_only": False, - } - ] - - -def test_temporary_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "source": "test_temp_volume", - "type": "volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - - -def test_docker_volume_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume_missing_volume_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} - with pytest.raises(Exception): - c1.add_storage("/some/path", vol_config) - - -def test_docker_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/some/path", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["volumes"] == {"test_volume": {}} - - -def test_anonymous_volume(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} - c1.add_storage("/some/path", vol_config) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} - ] - assert "volumes" not in output - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_not_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(read_only=False) - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": False, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] - - -def test_add_docker_socket_mount_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.storage.add_docker_socket(mount_path="/some/path") - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/some/path", - "read_only": True, - "bind": {"create_host_path": False, "propagation": "rprivate"}, - } - ] diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/validations.py b/ix-dev/community/webdav/templates/library/base_v2_0_21/validations.py deleted file mode 100644 index 57e039917b..0000000000 --- a/ix-dev/community/webdav/templates/library/base_v2_0_21/validations.py +++ /dev/null @@ -1,203 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/__init__.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/__init__.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/configs.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/configs.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/container.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/depends.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/depends.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/deploy.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/deploy.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/deps.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/device.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/device.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/devices.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/dns.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/dns.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/environment.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/error.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/error.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/formatter.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/formatter.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/functions.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/labels.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/labels.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/notes.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/notes.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/portal.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/portal.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/portals.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/portals.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/ports.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/ports.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/render.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/render.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/resources.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/resources.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/restart.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/restart.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/storage.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/__init__.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/tests/__init__.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_build_image.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_build_image.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_configs.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_configs.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_depends.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_depends.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_dns.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_dns.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_formatter.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_formatter.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_labels.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_labels.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_notes.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_notes.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_portal.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_portal.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_ports.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_ports.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_render.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_render.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_resources.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_resources.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_restart.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/tests/test_restart.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/webdav/templates/library/base_v2_1_0/validations.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/webdav/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/volume_mount.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/volume_mount.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/volume_mount_types.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/volume_mount_types.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/volume_sources.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/volume_sources.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/volume_types.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/volume_types.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/webdav/templates/library/base_v2_0_21/volumes.py b/ix-dev/community/webdav/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/webdav/templates/library/base_v2_0_21/volumes.py rename to ix-dev/community/webdav/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/whoogle/app.yaml b/ix-dev/community/whoogle/app.yaml index b9dc31e058..bc8eb4bc95 100644 --- a/ix-dev/community/whoogle/app.yaml +++ b/ix-dev/community/whoogle/app.yaml @@ -9,8 +9,8 @@ icon: https://media.sys.truenas.net/apps/whoogle/icons/icon.png keywords: - search - engine -lib_version: 2.0.31 -lib_version_hash: e61b4db536830d207e591fea73037ca3f335da01a7e4073bb37392d1ede15873 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -30,4 +30,4 @@ sources: - https://hub.docker.com/r/benbusby/whoogle-search title: Whoogle train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/container.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v2_0_31/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/deps.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v2_0_31/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/devices.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v2_0_31/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/environment.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v2_0_31/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/functions.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/functions.py deleted file mode 100644 index 47a2c1233a..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v2_0_31/functions.py +++ /dev/null @@ -1,148 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _or_default(self, value, default): - if not value: - return default - return value - - def _temp_config(self, name): - if not name: - raise RenderError("Expected [name] to be set when calling [temp_config].") - return {"type": "temporary", "volume_config": {"volume_name": name}} - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - "or_default": self._or_default, - "temp_config": self._temp_config, - } diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/healthcheck.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v2_0_31/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_container.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_deps.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_device.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_environment.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_healthcheck.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/validations.py b/ix-dev/community/whoogle/templates/library/base_v2_0_31/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/whoogle/templates/library/base_v2_0_31/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/__init__.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/__init__.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/configs.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/configs.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/container.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/depends.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/depends.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/deploy.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/deploy.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/deps.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/device.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/device.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/devices.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/dns.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/dns.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/environment.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/error.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/error.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/formatter.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/formatter.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/functions.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/labels.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/labels.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/notes.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/notes.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/portal.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/portal.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/portals.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/portals.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/ports.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/ports.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/render.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/render.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/resources.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/resources.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/restart.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/restart.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/storage.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/__init__.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/__init__.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_build_image.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_build_image.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_configs.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_configs.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_depends.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_depends.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_dns.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_dns.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_formatter.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_formatter.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_labels.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_labels.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_notes.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_notes.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_portal.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_portal.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_ports.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_ports.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_render.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_render.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_resources.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_resources.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_restart.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/tests/test_restart.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/whoogle/templates/library/base_v2_1_0/validations.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/whoogle/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/volume_mount.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/volume_mount.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/volume_mount_types.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/volume_mount_types.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/volume_sources.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/volume_sources.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/volume_types.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/volume_types.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/whoogle/templates/library/base_v2_0_31/volumes.py b/ix-dev/community/whoogle/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/whoogle/templates/library/base_v2_0_31/volumes.py rename to ix-dev/community/whoogle/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/wordpress/app.yaml b/ix-dev/community/wordpress/app.yaml index 5b110e80f8..d0e5dfc8f3 100644 --- a/ix-dev/community/wordpress/app.yaml +++ b/ix-dev/community/wordpress/app.yaml @@ -11,8 +11,8 @@ icon: https://media.sys.truenas.net/apps/wordpress/icons/icon.png keywords: - cms - blog -lib_version: 2.0.31 -lib_version_hash: e61b4db536830d207e591fea73037ca3f335da01a7e4073bb37392d1ede15873 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -38,4 +38,4 @@ sources: - https://hub.docker.com/_/wordpress title: Wordpress train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/container.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/container.py deleted file mode 100644 index a95e76734c..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v2_0_31/container.py +++ /dev/null @@ -1,317 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def add_snd_device(self): - self.add_group(29) - self.devices._add_snd_device() - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = f"{self._grace_period}s" - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_gpus(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/deps.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/deps.py deleted file mode 100644 index b3607fa6ab..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v2_0_31/deps.py +++ /dev/null @@ -1,454 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - recursive = action_config.get("recursive", False) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - recursive = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "recursive": recursive, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod, recursive=False): - print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chmod(os.path.join(root, f), int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid, recursive=False): - print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - if recursive: - for root, dirs, files in os.walk(path): - for f in files: - os.chown(os.path.join(root, f), uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"], action["recursive"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/devices.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/devices.py deleted file mode 100644 index ae22c79d2e..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v2_0_31/devices.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def _add_snd_device(self): - self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/environment.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v2_0_31/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/functions.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/functions.py deleted file mode 100644 index 47a2c1233a..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v2_0_31/functions.py +++ /dev/null @@ -1,148 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _or_default(self, value, default): - if not value: - return default - return value - - def _temp_config(self, name): - if not name: - raise RenderError("Expected [name] to be set when calling [temp_config].") - return {"type": "temporary", "volume_config": {"volume_name": name}} - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - "or_default": self._or_default, - "temp_config": self._temp_config, - } diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/healthcheck.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/healthcheck.py deleted file mode 100644 index 36ae5d90aa..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v2_0_31/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_container.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_container.py deleted file mode 100644 index 61a22a5df2..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_container.py +++ /dev/null @@ -1,324 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == "10s" - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_snd_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_snd_device() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] - assert output["services"]["test_container"]["group_add"] == [29, 568] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_deps.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_deps.py deleted file mode 100644 index f9562ba4f2..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_device.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_environment.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_healthcheck.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_healthcheck.py deleted file mode 100644 index fbd488ece4..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/validations.py b/ix-dev/community/wordpress/templates/library/base_v2_0_31/validations.py deleted file mode 100644 index 13f155dfdb..0000000000 --- a/ix-dev/community/wordpress/templates/library/base_v2_0_31/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb", "/dev/snd"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/__init__.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/__init__.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/configs.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/configs.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/container.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/depends.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/depends.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/deploy.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/deploy.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/deps.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/device.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/device.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/devices.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/dns.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/dns.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/environment.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/error.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/error.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/formatter.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/formatter.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/functions.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/labels.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/labels.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/notes.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/notes.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/portal.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/portal.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/portals.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/portals.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/ports.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/ports.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/render.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/render.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/resources.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/resources.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/restart.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/restart.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/storage.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/__init__.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/__init__.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_build_image.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_build_image.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_configs.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_configs.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_depends.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_depends.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_dns.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_dns.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_formatter.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_formatter.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_labels.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_labels.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_notes.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_notes.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_portal.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_portal.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_ports.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_ports.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_render.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_render.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_resources.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_resources.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_restart.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/tests/test_restart.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/wordpress/templates/library/base_v2_1_0/validations.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/wordpress/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_mount.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_mount.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_mount_types.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_mount_types.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_sources.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_sources.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_types.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/volume_types.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/wordpress/templates/library/base_v2_0_31/volumes.py b/ix-dev/community/wordpress/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/wordpress/templates/library/base_v2_0_31/volumes.py rename to ix-dev/community/wordpress/templates/library/base_v2_1_0/volumes.py diff --git a/ix-dev/community/zerotier/app.yaml b/ix-dev/community/zerotier/app.yaml index 09cc46a301..3f09c2dded 100644 --- a/ix-dev/community/zerotier/app.yaml +++ b/ix-dev/community/zerotier/app.yaml @@ -34,8 +34,8 @@ icon: https://media.sys.truenas.net/apps/zerotier/icons/icon.png keywords: - vpn - zerotier -lib_version: 2.0.25 -lib_version_hash: 37aa2a7f78f8ef1e8f3d705f4ce5d8bb66eddd4f023c25294a53a893b21a6048 +lib_version: 2.1.0 +lib_version_hash: 6f8024bfc9f06424dcc9dd3c0391d1b28afb5dddd5dcde2f6a9a0f8de2f74291 maintainers: - email: dev@ixsystems.com name: truenas @@ -53,4 +53,4 @@ sources: - https://hub.docker.com/r/zerotier/zerotier title: Zerotier train: community -version: 1.1.0 +version: 1.1.1 diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/container.py b/ix-dev/community/zerotier/templates/library/base_v2_0_25/container.py deleted file mode 100644 index 440926dd5b..0000000000 --- a/ix-dev/community/zerotier/templates/library/base_v2_0_25/container.py +++ /dev/null @@ -1,313 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .configs import ContainerConfigs - from .depends import Depends - from .deploy import Deploy - from .devices import Devices - from .dns import Dns - from .environment import Environment - from .error import RenderError - from .formatter import escape_dollar, get_image_with_hashed_data - from .healthcheck import Healthcheck - from .labels import Labels - from .ports import Ports - from .restart import RestartPolicy - from .validations import valid_network_mode_or_raise, valid_cap_or_raise - from .storage import Storage - from .sysctls import Sysctls -except ImportError: - from configs import ContainerConfigs - from depends import Depends - from deploy import Deploy - from devices import Devices - from dns import Dns - from environment import Environment - from error import RenderError - from formatter import escape_dollar, get_image_with_hashed_data - from healthcheck import Healthcheck - from labels import Labels - from ports import Ports - from restart import RestartPolicy - from validations import valid_network_mode_or_raise, valid_cap_or_raise - from storage import Storage - from sysctls import Sysctls - - -class Container: - def __init__(self, render_instance: "Render", name: str, image: str): - self._render_instance = render_instance - - self._name: str = name - self._image: str = self._resolve_image(image) - self._build_image: str = "" - self._user: str = "" - self._tty: bool = False - self._stdin_open: bool = False - self._init: bool | None = None - self._read_only: bool | None = None - self._hostname: str = "" - self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly - self._cap_add: set[str] = set() - self._security_opt: set[str] = set(["no-new-privileges"]) - self._group_add: set[int | str] = set() - self._network_mode: str = "" - self._entrypoint: list[str] = [] - self._command: list[str] = [] - self._grace_period: int | None = None - self._shm_size: int | None = None - self._storage: Storage = Storage(self._render_instance) - self.sysctls: Sysctls = Sysctls(self._render_instance, self) - self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) - self.deploy: Deploy = Deploy(self._render_instance) - self.networks: set[str] = set() - self.devices: Devices = Devices(self._render_instance) - self.environment: Environment = Environment(self._render_instance, self.deploy.resources) - self.dns: Dns = Dns(self._render_instance) - self.depends: Depends = Depends(self._render_instance) - self.healthcheck: Healthcheck = Healthcheck(self._render_instance) - self.labels: Labels = Labels(self._render_instance) - self.restart: RestartPolicy = RestartPolicy(self._render_instance) - self.ports: Ports = Ports(self._render_instance) - - self._auto_set_network_mode() - self._auto_add_labels() - self._auto_add_groups() - - def _auto_add_groups(self): - self.add_group(568) - - def _auto_set_network_mode(self): - if self._render_instance.values.get("network", {}).get("host_network", False): - self.set_network_mode("host") - - def _auto_add_labels(self): - labels = self._render_instance.values.get("labels", []) - if not labels: - return - - for label in labels: - containers = label.get("containers", []) - if not containers: - raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') - - if self._name in containers: - self.labels.add_label(label["key"], label["value"]) - - def _resolve_image(self, image: str): - images = self._render_instance.values["images"] - if image not in images: - raise RenderError( - f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" - ) - repo = images[image].get("repository", "") - tag = images[image].get("tag", "") - - if not repo: - raise RenderError(f"Repository not found for image [{image}]") - if not tag: - raise RenderError(f"Tag not found for image [{image}]") - - return f"{repo}:{tag}" - - def build_image(self, content: list[str | None]): - dockerfile = f"FROM {self._image}\n" - for line in content: - if not line: - continue - if line.startswith("FROM"): - # TODO: This will also block multi-stage builds - # We can revisit this later if we need it - raise RenderError( - "FROM cannot be used in build image. Define the base image when creating the container." - ) - dockerfile += line + "\n" - - self._build_image = dockerfile - self._image = get_image_with_hashed_data(self._image, dockerfile) - - def set_user(self, user: int, group: int): - for i in (user, group): - if not isinstance(i, int) or i < 0: - raise RenderError(f"User/Group [{i}] is not valid") - self._user = f"{user}:{group}" - - def add_group(self, group: int | str): - if isinstance(group, str): - group = str(group).strip() - if group.isdigit(): - raise RenderError(f"Group is a number [{group}] but passed as a string") - - if group in self._group_add: - raise RenderError(f"Group [{group}] already added") - self._group_add.add(group) - - def get_current_groups(self) -> list[str]: - return [str(g) for g in self._group_add] - - def set_tty(self, enabled: bool = False): - self._tty = enabled - - def set_stdin(self, enabled: bool = False): - self._stdin_open = enabled - - def set_init(self, enabled: bool = False): - self._init = enabled - - def set_read_only(self, enabled: bool = False): - self._read_only = enabled - - def set_hostname(self, hostname: str): - self._hostname = hostname - - def set_grace_period(self, grace_period: int): - if grace_period < 0: - raise RenderError(f"Grace period [{grace_period}] cannot be negative") - self._grace_period = grace_period - - def add_caps(self, caps: list[str]): - for c in caps: - if c in self._cap_add: - raise RenderError(f"Capability [{c}] already added") - self._cap_add.add(valid_cap_or_raise(c)) - - def add_security_opt(self, opt: str): - if opt in self._security_opt: - raise RenderError(f"Security Option [{opt}] already added") - self._security_opt.add(opt) - - def remove_security_opt(self, opt: str): - self._security_opt.remove(opt) - - def set_network_mode(self, mode: str): - self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) - - def set_entrypoint(self, entrypoint: list[str]): - self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] - - def set_command(self, command: list[str]): - self._command = [escape_dollar(str(e)) for e in command] - - def add_storage(self, mount_path: str, config: "IxStorage"): - self._storage.add(mount_path, config) - - def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): - self.add_group(999) - self._storage._add_docker_socket(read_only, mount_path) - - def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): - self._storage._add_tun_device(read_only, mount_path) - - def set_shm_size_mb(self, size: int): - self._shm_size = size - - # Easily remove devices from the container - # Useful in dependencies like postgres and redis - # where there is no need to pass devices to them - def remove_devices(self): - self.deploy.resources.remove_devices() - self.devices.remove_devices() - - @property - def storage(self): - return self._storage - - def render(self) -> dict[str, Any]: - if self._network_mode and self.networks: - raise RenderError("Cannot set both [network_mode] and [networks]") - - result = { - "image": self._image, - "platform": "linux/amd64", - "tty": self._tty, - "stdin_open": self._stdin_open, - "restart": self.restart.render(), - "cap_drop": sorted(self._cap_drop), - "healthcheck": self.healthcheck.render(), - } - - if self._hostname: - result["hostname"] = self._hostname - - if self._build_image: - result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} - - if self.configs.has_configs(): - result["configs"] = self.configs.render() - - if self._init is not None: - result["init"] = self._init - - if self._read_only is not None: - result["read_only"] = self._read_only - - if self._grace_period is not None: - result["stop_grace_period"] = self._grace_period - - if self._user: - result["user"] = self._user - - if self.deploy.resources.has_gpus() or self.devices.has_devices(): - self.add_group(44) # video - self.add_group(107) # render - - if self._group_add: - result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) - - if self._shm_size is not None: - result["shm_size"] = f"{self._shm_size}M" - - if self._cap_add: - result["cap_add"] = sorted(self._cap_add) - - if self._security_opt: - result["security_opt"] = sorted(self._security_opt) - - if self._network_mode: - result["network_mode"] = self._network_mode - - if self.sysctls.has_sysctls(): - result["sysctls"] = self.sysctls.render() - - if self._network_mode != "host": - if self.ports.has_ports(): - result["ports"] = self.ports.render() - - if self._entrypoint: - result["entrypoint"] = self._entrypoint - - if self._command: - result["command"] = self._command - - if self.devices.has_devices(): - result["devices"] = self.devices.render() - - if self.deploy.has_deploy(): - result["deploy"] = self.deploy.render() - - if self.environment.has_variables(): - result["environment"] = self.environment.render() - - if self.labels.has_labels(): - result["labels"] = self.labels.render() - - if self.dns.has_dns_nameservers(): - result["dns"] = self.dns.render_dns_nameservers() - - if self.dns.has_dns_searches(): - result["dns_search"] = self.dns.render_dns_searches() - - if self.dns.has_dns_opts(): - result["dns_opt"] = self.dns.render_dns_opts() - - if self.depends.has_dependencies(): - result["depends_on"] = self.depends.render() - - if self._storage.has_mounts(): - result["volumes"] = self._storage.render() - - return result diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/deps.py b/ix-dev/community/zerotier/templates/library/base_v2_0_25/deps.py deleted file mode 100644 index fdf69b2875..0000000000 --- a/ix-dev/community/zerotier/templates/library/base_v2_0_25/deps.py +++ /dev/null @@ -1,443 +0,0 @@ -import os -import json -import urllib.parse -from typing import TYPE_CHECKING, TypedDict, NotRequired - -if TYPE_CHECKING: - from render import Render - from storage import IxStorage - -try: - from .error import RenderError - from .validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) -except ImportError: - from error import RenderError - from validations import ( - valid_port_or_raise, - valid_fs_path_or_raise, - valid_octal_mode_or_raise, - valid_redis_password_or_raise, - ) - - -class PostgresConfig(TypedDict): - user: str - password: str - database: str - port: NotRequired[int] - volume: "IxStorage" - - -class MariadbConfig(TypedDict): - user: str - password: str - database: str - root_password: NotRequired[str] - port: NotRequired[int] - auto_upgrade: NotRequired[bool] - volume: "IxStorage" - - -class RedisConfig(TypedDict): - password: str - port: NotRequired[int] - volume: "IxStorage" - - -class PermsContainer: - def __init__(self, render_instance: "Render", name: str): - self._render_instance = render_instance - self._name = name - self.actions: set[str] = set() - self.parsed_configs: list[dict] = [] - - def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - identifier = self.normalize_identifier_for_path(identifier) - if identifier in self.actions: - raise RenderError(f"Action with id [{identifier}] already used for another permission action") - - parsed_action = self.parse_action(identifier, volume_config, action_config) - if parsed_action: - self.parsed_configs.append(parsed_action) - self.actions.add(identifier) - - def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): - valid_modes = [ - "always", # Always set permissions, without checking. - "check", # Checks if permissions are correct, and set them if not. - ] - mode = action_config.get("mode", "check") - uid = action_config.get("uid", None) - gid = action_config.get("gid", None) - chmod = action_config.get("chmod", None) - mount_path = os.path.join("/mnt/permission", identifier) - is_temporary = False - - vol_type = volume_config.get("type", "") - match vol_type: - case "temporary": - # If it is a temporary volume, we force auto permissions - # and set is_temporary to True, so it will be cleaned up - is_temporary = True - case "volume": - if not volume_config.get("volume_config", {}).get("auto_permissions", False): - return None - case "host_path": - host_path_config = volume_config.get("host_path_config", {}) - # Skip when ACL enabled - if host_path_config.get("acl_enable", False): - return None - if not host_path_config.get("auto_permissions", False): - return None - case "ix_volume": - ix_vol_config = volume_config.get("ix_volume_config", {}) - # Skip when ACL enabled - if ix_vol_config.get("acl_enable", False): - return None - # For ix_volumes, we default to auto_permissions = True - if not ix_vol_config.get("auto_permissions", True): - return None - case _: - # Skip for other types - return None - - if mode not in valid_modes: - raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") - if not isinstance(uid, int) or not isinstance(gid, int): - raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") - if chmod is not None: - chmod = valid_octal_mode_or_raise(chmod) - - mount_path = valid_fs_path_or_raise(mount_path) - return { - "mount_path": mount_path, - "volume_config": volume_config, - "action_data": { - "mount_path": mount_path, - "is_temporary": is_temporary, - "identifier": identifier, - "mode": mode, - "uid": uid, - "gid": gid, - "chmod": chmod, - }, - } - - def normalize_identifier_for_path(self, identifier: str): - return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") - - def has_actions(self): - return bool(self.actions) - - def activate(self): - if len(self.parsed_configs) != len(self.actions): - raise RenderError("Number of actions and parsed configs does not match") - - if not self.has_actions(): - raise RenderError("No actions added. Check if there are actions before activating") - - # Add the container and set it up - c = self._render_instance.add_container(self._name, "python_permissions_image") - c.set_user(0, 0) - c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) - c.set_network_mode("none") - - # Don't attach any devices - c.remove_devices() - - c.deploy.resources.set_profile("medium") - c.restart.set_policy("on-failure", maximum_retry_count=1) - c.healthcheck.disable() - - c.set_entrypoint(["python3", "/script/run.py"]) - script = "#!/usr/bin/env python3\n" - script += get_script() - c.configs.add("permissions_run_script", script, "/script/run.py", "0700") - - actions_data: list[dict] = [] - for parsed in self.parsed_configs: - c.add_storage(parsed["mount_path"], parsed["volume_config"]) - actions_data.append(parsed["action_data"]) - - actions_data_json = json.dumps(actions_data) - c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") - - -def get_script(): - return """ -import os -import json -import time -import shutil - -with open("/script/actions.json", "r") as f: - actions_data = json.load(f) - -if not actions_data: - # If this script is called, there should be actions data - raise ValueError("No actions data found") - -def fix_perms(path, chmod): - print(f"Changing permissions to {chmod} on: [{path}]") - os.chmod(path, int(chmod, 8)) - print("Permissions after changes:") - print_chmod_stat() - -def fix_owner(path, uid, gid): - print(f"Changing ownership to {uid}:{gid} on: [{path}]") - os.chown(path, uid, gid) - print("Ownership after changes:") - print_chown_stat() - -def print_chown_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") - -def print_chmod_stat(): - curr_stat = os.stat(action["mount_path"]) - print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") - -def print_chown_diff(curr_stat, uid, gid): - print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") - -def print_chmod_diff(curr_stat, mode): - print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") - -def perform_action(action): - start_time = time.time() - print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") - - if not os.path.isdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not a directory, skipping...") - return - - if action["is_temporary"]: - print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") - for item in os.listdir(action["mount_path"]): - item_path = os.path.join(action["mount_path"], item) - - # Exclude the safe directory, where we can use to mount files temporarily - if os.path.basename(item_path) == "ix-safe": - continue - if os.path.isdir(item_path): - shutil.rmtree(item_path) - else: - os.remove(item_path) - - if not action["is_temporary"] and os.listdir(action["mount_path"]): - print(f"Path [{action['mount_path']}] is not empty, skipping...") - return - - print(f"Current Ownership and Permissions on [{action['mount_path']}]:") - curr_stat = os.stat(action["mount_path"]) - print_chown_diff(curr_stat, action["uid"], action["gid"]) - print_chmod_diff(curr_stat, action["chmod"]) - print("---") - - if action["mode"] == "always": - fix_owner(action["mount_path"], action["uid"], action["gid"]) - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - fix_perms(action["mount_path"], action["chmod"]) - return - - elif action["mode"] == "check": - if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: - print("Ownership is incorrect. Fixing...") - fix_owner(action["mount_path"], action["uid"], action["gid"]) - else: - print("Ownership is correct. Skipping...") - - if not action["chmod"]: - print("Skipping permissions check, chmod is falsy") - else: - if oct(curr_stat.st_mode)[3:] != action["chmod"]: - print("Permissions are incorrect. Fixing...") - fix_perms(action["mount_path"], action["chmod"]) - else: - print("Permissions are correct. Skipping...") - - print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") - print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") - print() - -if __name__ == "__main__": - start_time = time.time() - for action in actions_data: - perform_action(action) - print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") -""" - - -class Deps: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def perms(self, name: str): - return PermsContainer(self._render_instance, name) - - def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): - return PostgresContainer(self._render_instance, name, image, config, perms_instance) - - def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): - return RedisContainer(self._render_instance, name, image, config, perms_instance) - - def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): - return MariadbContainer(self._render_instance, name, image, config, perms_instance) - - -class PostgresContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for postgres") - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("postgres") - c.remove_devices() - - c.add_storage("/var/lib/postgresql/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("POSTGRES_USER", config["user"]) - c.environment.add_env("POSTGRES_PASSWORD", config["password"]) - c.environment.add_env("POSTGRES_DB", config["database"]) - c.environment.add_env("POSTGRES_PORT", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container - - def _get_port(self): - return self._config.get("port") or 5432 - - def get_url(self, variant: str): - user = urllib.parse.quote_plus(self._config["user"]) - password = urllib.parse.quote_plus(self._config["password"]) - creds = f"{user}:{password}" - addr = f"{self._name}:{self._get_port()}" - db = self._config["database"] - - match variant: - case "postgres": - return f"postgres://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql": - return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" - case "postgresql_no_creds": - return f"postgresql://{addr}/{db}?sslmode=disable" - case "host_port": - return addr - case _: - raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") - - -class RedisContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - self._config = config - - for key in ("password", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for redis") - - valid_redis_password_or_raise(config["password"]) - - port = valid_port_or_raise(self._get_port()) - - c = self._render_instance.add_container(name, image) - c.set_user(1001, 0) - c.healthcheck.set_test("redis") - c.remove_devices() - - c.add_storage("/bitnami/redis/data", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} - ) - - c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") - c.environment.add_env("REDIS_PASSWORD", config["password"]) - c.environment.add_env("REDIS_PORT_NUMBER", port) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - def _get_port(self): - return self._config.get("port") or 6379 - - def get_url(self, variant: str): - addr = f"{self._name}:{self._get_port()}" - password = urllib.parse.quote_plus(self._config["password"]) - - match variant: - case "redis": - return f"redis://default:{password}@{addr}" - - @property - def container(self): - return self._container - - -class MariadbContainer: - def __init__( - self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer - ): - self._render_instance = render_instance - self._name = name - - for key in ("user", "password", "database", "volume"): - if key not in config: - raise RenderError(f"Expected [{key}] to be set for mariadb") - - port = valid_port_or_raise(config.get("port") or 3306) - root_password = config.get("root_password") or config["password"] - auto_upgrade = config.get("auto_upgrade", True) - - c = self._render_instance.add_container(name, image) - c.set_user(999, 999) - c.healthcheck.set_test("mariadb") - c.remove_devices() - - c.add_storage("/var/lib/mysql", config["volume"]) - perms_instance.add_or_skip_action( - f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} - ) - - c.environment.add_env("MARIADB_USER", config["user"]) - c.environment.add_env("MARIADB_PASSWORD", config["password"]) - c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) - c.environment.add_env("MARIADB_DATABASE", config["database"]) - c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) - c.set_command(["--port", str(port)]) - - # Store container for further configuration - # For example: c.depends.add_dependency("other_container", "service_started") - self._container = c - - @property - def container(self): - return self._container diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/devices.py b/ix-dev/community/zerotier/templates/library/base_v2_0_25/devices.py deleted file mode 100644 index c9f8cf633e..0000000000 --- a/ix-dev/community/zerotier/templates/library/base_v2_0_25/devices.py +++ /dev/null @@ -1,63 +0,0 @@ -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .device import Device -except ImportError: - from error import RenderError - from device import Device - - -class Devices: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._devices: set[Device] = set() - - # Tracks all container device paths to make sure they are not duplicated - self._container_device_paths: set[str] = set() - # Scan values for devices we should automatically add - # for example /dev/dri for gpus - self._auto_add_devices_from_values() - - def _auto_add_devices_from_values(self): - resources = self._render_instance.values.get("resources", {}) - - if resources.get("gpus", {}).get("use_all_gpus", False): - self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) - - def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): - # Host device can be mapped to multiple container devices, - # so we only make sure container devices are not duplicated - if container_device in self._container_device_paths: - raise RenderError(f"Device with container path [{container_device}] already added") - - self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) - self._container_device_paths.add(container_device) - - def add_usb_bus(self): - self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) - - def has_devices(self): - return len(self._devices) > 0 - - # Mainly will be used from dependencies - # There is no reason to pass devices to - # redis or postgres for example - def remove_devices(self): - self._devices.clear() - self._container_device_paths.clear() - - # Check if there are any gpu devices - # Used to determine if we should add groups - # like 'video' to the container - def has_gpus(self): - for d in self._devices: - if d.host_device == "/dev/dri": - return True - return False - - def render(self) -> list[str]: - return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/environment.py b/ix-dev/community/zerotier/templates/library/base_v2_0_25/environment.py deleted file mode 100644 index 850a3afd8e..0000000000 --- a/ix-dev/community/zerotier/templates/library/base_v2_0_25/environment.py +++ /dev/null @@ -1,109 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render -try: - from .error import RenderError - from .formatter import escape_dollar - from .resources import Resources -except ImportError: - from error import RenderError - from formatter import escape_dollar - from resources import Resources - - -class Environment: - def __init__(self, render_instance: "Render", resources: Resources): - self._render_instance = render_instance - self._resources = resources - # Stores variables that user defined - self._user_vars: dict[str, Any] = {} - # Stores variables that are automatically added (based on values) - self._auto_variables: dict[str, Any] = {} - # Stores variables that are added by the application developer - self._app_dev_variables: dict[str, Any] = {} - - self._auto_add_variables_from_values() - - def _auto_add_variables_from_values(self): - self._add_generic_variables() - self._add_nvidia_variables() - - def _add_generic_variables(self): - self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") - self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") - self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") - - run_as = self._render_instance.values.get("run_as", {}) - user = run_as.get("user") - group = run_as.get("group") - if user: - self._auto_variables["PUID"] = user - self._auto_variables["UID"] = user - self._auto_variables["USER_ID"] = user - if group: - self._auto_variables["PGID"] = group - self._auto_variables["GID"] = group - self._auto_variables["GROUP_ID"] = group - - def _add_nvidia_variables(self): - if self._resources._nvidia_ids: - self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) - else: - self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" - - def _format_value(self, v: Any) -> str: - value = str(v) - - # str(bool) returns "True" or "False", - # but we want "true" or "false" - if isinstance(v, bool): - value = value.lower() - return value - - def add_env(self, name: str, value: Any): - if not name: - raise RenderError(f"Environment variable name cannot be empty. [{name}]") - if name in self._app_dev_variables.keys(): - raise RenderError( - f"Found duplicate environment variable [{name}] in application developer environment variables." - ) - self._app_dev_variables[name] = value - - def add_user_envs(self, user_env: list[dict]): - for item in user_env: - if not item.get("name"): - raise RenderError(f"Environment variable name cannot be empty. [{item}]") - if item["name"] in self._user_vars.keys(): - raise RenderError( - f"Found duplicate environment variable [{item['name']}] in user environment variables." - ) - self._user_vars[item["name"]] = item.get("value") - - def has_variables(self): - return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 - - def render(self): - result: dict[str, str] = {} - - # Add envs from auto variables - result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) - - # Track defined keys for faster lookup - defined_keys = set(result.keys()) - - # Add envs from application developer (prohibit overwriting auto variables) - for k, v in self._app_dev_variables.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") - result[k] = self._format_value(v) - defined_keys.add(k) - - # Add envs from user (prohibit overwriting app developer envs and auto variables) - for k, v in self._user_vars.items(): - if k in defined_keys: - raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") - result[k] = self._format_value(v) - - return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/functions.py b/ix-dev/community/zerotier/templates/library/base_v2_0_25/functions.py deleted file mode 100644 index 304dc1300f..0000000000 --- a/ix-dev/community/zerotier/templates/library/base_v2_0_25/functions.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import bcrypt -import secrets -from base64 import b64encode -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .volume_sources import HostPathSource, IxVolumeSource -except ImportError: - from error import RenderError - from volume_sources import HostPathSource, IxVolumeSource - - -class Functions: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - - def _bcrypt_hash(self, password): - hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") - return hashed - - def _htpasswd(self, username, password): - hashed = self._bcrypt_hash(password) - return username + ":" + hashed - - def _secure_string(self, length): - return secrets.token_urlsafe(length) - - def _basic_auth(self, username, password): - return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") - - def _basic_auth_header(self, username, password): - return f"Basic {self._basic_auth(username, password)}" - - def _fail(self, message): - raise RenderError(message) - - def _camel_case(self, string): - return string.title() - - def _auto_cast(self, value): - try: - return int(value) - except ValueError: - pass - - try: - return float(value) - except ValueError: - pass - - if value.lower() in ["true", "false"]: - return value.lower() == "true" - - return value - - def _match_regex(self, value, regex): - if not re.match(regex, value): - return False - return True - - def _must_match_regex(self, value, regex): - if not self._match_regex(value, regex): - raise RenderError(f"Expected [{value}] to match [{regex}]") - return value - - def _is_boolean(self, string): - return string.lower() in ["true", "false"] - - def _is_number(self, string): - try: - float(string) - return True - except ValueError: - return False - - def _copy_dict(self, dict): - return dict.copy() - - def _merge_dicts(self, *dicts): - merged_dict = {} - for dictionary in dicts: - merged_dict.update(dictionary) - return merged_dict - - def _disallow_chars(self, string: str, chars: list[str], key: str): - for char in chars: - if char in string: - raise RenderError(f"Disallowed character [{char}] in [{key}]") - return string - - def _get_host_path(self, storage): - source_type = storage.get("type", "") - if not source_type: - raise RenderError("Expected [type] to be set for volume mounts.") - - match source_type: - case "host_path": - mount_config = storage.get("host_path_config") - if mount_config is None: - raise RenderError("Expected [host_path_config] to be set for [host_path] type.") - host_source = HostPathSource(self._render_instance, mount_config).get() - return host_source - case "ix_volume": - mount_config = storage.get("ix_volume_config") - if mount_config is None: - raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") - ix_source = IxVolumeSource(self._render_instance, mount_config).get() - return ix_source - case _: - raise RenderError(f"Storage type [{source_type}] does not support host path.") - - def func_map(self): - # TODO: Check what is no longer used and remove - return { - "auto_cast": self._auto_cast, - "basic_auth_header": self._basic_auth_header, - "basic_auth": self._basic_auth, - "bcrypt_hash": self._bcrypt_hash, - "camel_case": self._camel_case, - "copy_dict": self._copy_dict, - "fail": self._fail, - "htpasswd": self._htpasswd, - "is_boolean": self._is_boolean, - "is_number": self._is_number, - "match_regex": self._match_regex, - "merge_dicts": self._merge_dicts, - "must_match_regex": self._must_match_regex, - "secure_string": self._secure_string, - "disallow_chars": self._disallow_chars, - "get_host_path": self._get_host_path, - } diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/healthcheck.py b/ix-dev/community/zerotier/templates/library/base_v2_0_25/healthcheck.py deleted file mode 100644 index a54f3f3133..0000000000 --- a/ix-dev/community/zerotier/templates/library/base_v2_0_25/healthcheck.py +++ /dev/null @@ -1,193 +0,0 @@ -from typing import Any, TYPE_CHECKING - -if TYPE_CHECKING: - from render import Render - -try: - from .error import RenderError - from .formatter import escape_dollar - from .validations import valid_http_path_or_raise -except ImportError: - from error import RenderError - from formatter import escape_dollar - from validations import valid_http_path_or_raise - - -class Healthcheck: - def __init__(self, render_instance: "Render"): - self._render_instance = render_instance - self._test: str | list[str] = "" - self._interval_sec: int = 10 - self._timeout_sec: int = 5 - self._retries: int = 30 - self._start_period_sec: int = 10 - self._disabled: bool = False - - def _get_test(self): - if isinstance(self._test, str): - return escape_dollar(self._test) - - return [escape_dollar(t) for t in self._test] - - def disable(self): - self._disabled = True - - def set_custom_test(self, test: str | list[str]): - if self._disabled: - raise RenderError("Cannot set custom test when healthcheck is disabled") - self._test = test - - def set_test(self, variant: str, config: dict | None = None): - config = config or {} - self.set_custom_test(test_mapping(variant, config)) - - def set_interval(self, interval: int): - self._interval_sec = interval - - def set_timeout(self, timeout: int): - self._timeout_sec = timeout - - def set_retries(self, retries: int): - self._retries = retries - - def set_start_period(self, start_period: int): - self._start_period_sec = start_period - - def render(self): - if self._disabled: - return {"disable": True} - - if not self._test: - raise RenderError("Healthcheck test is not set") - - return { - "test": self._get_test(), - "interval": f"{self._interval_sec}s", - "timeout": f"{self._timeout_sec}s", - "retries": self._retries, - "start_period": f"{self._start_period_sec}s", - } - - -def test_mapping(variant: str, config: dict | None = None) -> str: - config = config or {} - tests = { - "curl": curl_test, - "wget": wget_test, - "http": http_test, - "netcat": netcat_test, - "tcp": tcp_test, - "redis": redis_test, - "postgres": postgres_test, - "mariadb": mariadb_test, - } - - if variant not in tests: - raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") - - return tests[variant](config) - - -def get_key(config: dict, key: str, default: Any, required: bool): - if not config.get(key): - if not required: - return default - raise RenderError(f"Expected [{key}] to be set") - return config[key] - - -def curl_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--insecure") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for curl test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "curl --silent --output /dev/null --show-error --fail" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def wget_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - scheme = get_key(config, "scheme", "http", False) - host = get_key(config, "host", "127.0.0.1", False) - headers = get_key(config, "headers", [], False) - - opts = [] - if scheme == "https": - opts.append("--no-check-certificate") - - for header in headers: - if not header[0] or not header[1]: - raise RenderError("Expected [header] to be a list of two items for wget test") - opts.append(f'--header "{header[0]}: {header[1]}"') - - cmd = "wget --spider --quiet" - if opts: - cmd += f" {' '.join(opts)}" - cmd += f" {scheme}://{host}:{port}{path}" - return cmd - - -def http_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - path = valid_http_path_or_raise(get_key(config, "path", "/", False)) - host = get_key(config, "host", "127.0.0.1", False) - - return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep -q "200 OK"'""" # noqa - - -def netcat_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"nc -z -w 1 {host} {port}" - - -def tcp_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", None, True) - host = get_key(config, "host", "127.0.0.1", False) - - return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" - - -def redis_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 6379, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" - - -def postgres_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 5432, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" - - -def mariadb_test(config: dict) -> str: - config = config or {} - port = get_key(config, "port", 3306, False) - host = get_key(config, "host", "127.0.0.1", False) - - return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_container.py b/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_container.py deleted file mode 100644 index 84d712a403..0000000000 --- a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_container.py +++ /dev/null @@ -1,314 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_empty_container_name(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container(" ", "test_image") - - -def test_resolve_image(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["image"] == "nginx:latest" - - -def test_missing_repo(mock_values): - mock_values["images"]["test_image"]["repository"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_missing_tag(mock_values): - mock_values["images"]["test_image"]["tag"] = "" - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "test_image") - - -def test_non_existing_image(mock_values): - render = Render(mock_values) - with pytest.raises(Exception): - render.add_container("test_container", "non_existing_image") - - -def test_tty(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_tty(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["tty"] is True - - -def test_init(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_init(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["init"] is True - - -def test_read_only(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_read_only(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["read_only"] is True - - -def test_stdin(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_stdin(True) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stdin_open"] is True - - -def test_hostname(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_hostname("test_hostname") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["hostname"] == "test_hostname" - - -def test_grace_period(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_grace_period(10) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["stop_grace_period"] == 10 - - -def test_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_user(1000, 1000) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["user"] == "1000:1000" - - -def test_invalid_user(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_user(-100, 1000) - - -def test_add_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - c1.add_group("video") - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] - - -def test_add_duplicate_group(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_group(1000) - with pytest.raises(Exception): - c1.add_group(1000) - - -def test_add_group_as_string(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_group("1000") - - -def test_add_docker_socket(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_docker_socket() - output = render.render() - assert output["services"]["test_container"]["group_add"] == [568, 999] - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/var/run/docker.sock", - "target": "/var/run/docker.sock", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_tun_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_tun_device() - output = render.render() - assert output["services"]["test_container"]["volumes"] == [ - { - "type": "bind", - "source": "/dev/net/tun", - "target": "/dev/net/tun", - "read_only": True, - "bind": { - "propagation": "rprivate", - "create_host_path": False, - }, - } - ] - - -def test_shm_size(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_shm_size_mb(10) - output = render.render() - assert output["services"]["test_container"]["shm_size"] == "10M" - - -def test_valid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_caps(["ALL", "NET_ADMIN"]) - output = render.render() - assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] - assert output["services"]["test_container"]["cap_drop"] == ["ALL"] - - -def test_add_duplicate_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) - - -def test_invalid_caps(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_caps(["invalid_cap"]) - - -def test_remove_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.remove_security_opt("no-new-privileges") - output = render.render() - assert "security_opt" not in output["services"]["test_container"] - - -def test_add_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.add_security_opt("seccomp=unconfined") - output = render.render() - assert output["services"]["test_container"]["security_opt"] == [ - "no-new-privileges", - "seccomp=unconfined", - ] - - -def test_add_duplicate_security_opt(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.add_security_opt("no-new-privileges") - - -def test_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("host") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_auto_network_mode_with_host_network(mock_values): - mock_values["network"] = {"host_network": True} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "host" - - -def test_network_mode_with_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.set_network_mode("service:test_container") - output = render.render() - assert output["services"]["test_container"]["network_mode"] == "service:test_container" - - -def test_network_mode_with_container_missing(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("service:missing_container") - - -def test_invalid_network_mode(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.set_network_mode("invalid_mode") - - -def test_entrypoint(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] - - -def test_command(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.set_command(["echo", "hello $MY_ENV"]) - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_deps.py b/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_deps.py deleted file mode 100644 index 81e399c85d..0000000000 --- a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_deps.py +++ /dev/null @@ -1,380 +0,0 @@ -import json -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_postgres_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.postgres( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_postgres(mock_values): - mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - p = render.deps.postgres( - "pg_container", - "pg_image", - { - "user": "test_user", - "password": "test_@password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - p.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert ( - p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" - ) - assert "devices" not in output["services"]["pg_container"] - assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] - assert output["services"]["pg_container"]["image"] == "postgres:latest" - assert output["services"]["pg_container"]["user"] == "999:999" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["pg_container"]["healthcheck"] == { - "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["pg_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/postgresql/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["pg_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "POSTGRES_USER": "test_user", - "POSTGRES_PASSWORD": "test_@password", - "POSTGRES_DB": "test_database", - "POSTGRES_PORT": "5432", - } - assert output["services"]["pg_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - assert output["services"]["perms_container"]["restart"] == "on-failure:1" - - -def test_add_redis_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test_password", "volume": {}}, # type: ignore - ) - - -def test_add_redis_with_password_with_spaces(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.redis( - "test_container", - "test_image", - {"password": "test password", "volume": {}}, # type: ignore - ) - - -def test_add_redis(mock_values): - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - r = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test&password@", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - c1.environment.add_env("REDIS_URL", r.get_url("redis")) - if perms_container.has_actions(): - perms_container.activate() - r.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["redis_container"] - assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] - assert ( - output["services"]["test_container"]["environment"]["REDIS_URL"] - == "redis://default:test%26password%40@redis_container:6379" - ) - assert output["services"]["redis_container"]["image"] == "redis:latest" - assert output["services"]["redis_container"]["user"] == "1001:0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["redis_container"]["healthcheck"] == { - "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["redis_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/bitnami/redis/data", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["redis_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "ALLOW_EMPTY_PASSWORD": "no", - "REDIS_PASSWORD": "test&password@", - "REDIS_PORT_NUMBER": "6379", - } - assert output["services"]["redis_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_mariadb_missing_config(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - render.deps.mariadb( - "test_container", - "test_image", - {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore - ) - - -def test_add_mariadb(mock_values): - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - perms_container = render.deps.perms("perms_container") - m = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - if perms_container.has_actions(): - perms_container.activate() - m.container.depends.add_dependency("perms_container", "service_completed_successfully") - output = render.render() - assert "devices" not in output["services"]["mariadb_container"] - assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] - assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" - assert output["services"]["mariadb_container"]["user"] == "999:999" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" - assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" - assert output["services"]["mariadb_container"]["healthcheck"] == { - "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - assert output["services"]["mariadb_container"]["volumes"] == [ - { - "type": "volume", - "source": "test_volume", - "target": "/var/lib/mysql", - "read_only": False, - "volume": {"nocopy": False}, - } - ] - assert output["services"]["mariadb_container"]["environment"] == { - "TZ": "Etc/UTC", - "UMASK": "002", - "UMASK_SET": "002", - "NVIDIA_VISIBLE_DEVICES": "void", - "MARIADB_USER": "test_user", - "MARIADB_PASSWORD": "test_password", - "MARIADB_ROOT_PASSWORD": "test_password", - "MARIADB_DATABASE": "test_database", - "MARIADB_AUTO_UPGRADE": "true", - } - assert output["services"]["mariadb_container"]["depends_on"] == { - "perms_container": {"condition": "service_completed_successfully"} - } - - -def test_add_perms_container(mock_values): - mock_values["ix_volumes"] = { - "test_dataset1": "/mnt/test/1", - "test_dataset2": "/mnt/test/2", - "test_dataset3": "/mnt/test/3", - } - mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "latest"} - mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} - mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - # fmt: off - volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} - host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} - host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} - host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa - ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} - ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa - ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa - temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} - # fmt: on - - c1.add_storage("/some/path1", volume_perms) - c1.add_storage("/some/path2", volume_no_perms) - c1.add_storage("/some/path3", host_path_perms) - c1.add_storage("/some/path4", host_path_no_perms) - c1.add_storage("/some/path5", host_path_acl_perms) - c1.add_storage("/some/path6", ix_volume_no_perms) - c1.add_storage("/some/path7", ix_volume_perms) - c1.add_storage("/some/path8", ix_volume_acl_perms) - c1.add_storage("/some/path9", temp_volume) - - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) - perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) - - postgres = render.deps.postgres( - "postgres_container", - "postgres_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - redis = render.deps.redis( - "redis_container", - "redis_image", - { - "password": "test_password", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - mariadb = render.deps.mariadb( - "mariadb_container", - "mariadb_image", - { - "user": "test_user", - "password": "test_password", - "database": "test_database", - "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, - }, - perms_container, - ) - - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert output["services"]["test_perms_container"]["network_mode"] == "none" - assert output["services"]["test_container"]["depends_on"] == { - "test_perms_container": {"condition": "service_completed_successfully"} - } - assert output["configs"]["permissions_run_script"]["content"] != "" - # fmt: off - content = [ - {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa - {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa - ] - # fmt: on - assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) - - -def test_add_duplicate_perms_action(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - with pytest.raises(Exception): - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - - -def test_add_perm_action_without_auto_perms_enabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} - c1.add_storage("/some/path", vol_config) - perms_container = render.deps.perms("test_perms_container") - perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) - if perms_container.has_actions(): - perms_container.activate() - c1.depends.add_dependency("test_perms_container", "service_completed_successfully") - output = render.render() - assert "configs" not in output - assert "ix-test_perms_container" not in output["services"] - assert "depends_on" not in output["services"]["test_container"] diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_device.py b/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_device.py deleted file mode 100644 index 7455c829f6..0000000000 --- a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_device.py +++ /dev/null @@ -1,121 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_add_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] - - -def test_devices_without_host(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("", "/c/dev/sda") - - -def test_devices_without_container(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "") - - -def test_add_duplicate_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda") - - -def test_add_device_with_invalid_container_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "c/dev/sda") - - -def test_add_device_with_invalid_host_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("h/dev/sda", "/c/dev/sda") - - -def test_add_disallowed_device(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/dri", "/c/dev/sda") - - -def test_add_device_with_invalid_cgroup_perm(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") - - -def test_automatically_add_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] - assert output["services"]["test_container"]["group_add"] == [44, 107, 568] - - -def test_remove_gpu_devices(mock_values): - mock_values["resources"] = {"gpus": {"use_all_gpus": True}} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.remove_devices() - output = render.render() - assert "devices" not in output["services"]["test_container"] - assert output["services"]["test_container"]["group_add"] == [568] - - -def test_add_usb_bus(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.devices.add_usb_bus() - output = render.render() - assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] - - -def test_add_usb_bus_disallowed(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_environment.py b/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_environment.py deleted file mode 100644 index 209f67551b..0000000000 --- a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_environment.py +++ /dev/null @@ -1,184 +0,0 @@ -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_auto_add_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - mock_values["run_as"] = {"user": "1000", "group": "1000"} - mock_values["resources"] = { - "gpus": { - "nvidia_gpu_selection": { - "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, - "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, - }, - } - } - - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert len(envs) == 11 - assert envs["TZ"] == "Etc/UTC" - assert envs["PUID"] == "1000" - assert envs["UID"] == "1000" - assert envs["USER_ID"] == "1000" - assert envs["PGID"] == "1000" - assert envs["GID"] == "1000" - assert envs["GROUP_ID"] == "1000" - assert envs["UMASK"] == "002" - assert envs["UMASK_SET"] == "002" - assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" - assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" - - -def test_add_from_all_sources(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_value") - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_value" - assert envs["USER_ENV"] == "test_value2" - assert envs["TZ"] == "Etc/UTC" - - -def test_user_add_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV2", "value": "test_value2"}, - ] - ) - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["MY_ENV"] == "test_value" - assert envs["MY_ENV2"] == "test_value2" - - -def test_user_add_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "MY_ENV", "value": "test_value"}, - {"name": "MY_ENV", "value": "test_value2"}, - ] - ) - - -def test_user_env_without_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_user_envs( - [ - {"name": "", "value": "test_value"}, - ] - ) - - -def test_user_env_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "TZ", "value": "test_value"}, - ] - ) - with pytest.raises(Exception): - render.render() - - -def test_user_env_try_to_overwrite_app_dev_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_user_envs( - [ - {"name": "PORT", "value": "test_value"}, - ] - ) - c1.environment.add_env("PORT", "test_value2") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): - mock_values["TZ"] = "Etc/UTC" - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("TZ", "test_value") - with pytest.raises(Exception): - render.render() - - -def test_app_dev_no_name(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.environment.add_env("", "test_value") - - -def test_app_dev_duplicate_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("PORT", "test_value") - with pytest.raises(Exception): - c1.environment.add_env("PORT", "test_value2") - - -def test_format_vars(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - c1.environment.add_env("APP_ENV", "test_$value") - c1.environment.add_env("APP_ENV_BOOL", True) - c1.environment.add_env("APP_ENV_INT", 10) - c1.environment.add_env("APP_ENV_FLOAT", 10.5) - c1.environment.add_user_envs( - [ - {"name": "USER_ENV", "value": "test_$value2"}, - ] - ) - - output = render.render() - envs = output["services"]["test_container"]["environment"] - assert envs["APP_ENV"] == "test_$$value" - assert envs["USER_ENV"] == "test_$$value2" - assert envs["APP_ENV_BOOL"] == "true" - assert envs["APP_ENV_INT"] == "10" - assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_functions.py b/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_functions.py deleted file mode 100644 index 6857fce3f1..0000000000 --- a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_functions.py +++ /dev/null @@ -1,78 +0,0 @@ -import re -import pytest - - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_funcs(mock_values): - mock_values["ix_volumes"] = {"test": "/mnt/test123"} - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - - tests = [ - {"func": "auto_cast", "values": ["1"], "expected": 1}, - {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, - {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, - { - "func": "bcrypt_hash", - "values": ["my_pass"], - "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, - {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, - {"func": "fail", "values": ["my_message"], "expect_raise": True}, - { - "func": "htpasswd", - "values": ["my_user", "my_pass"], - "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", - }, - {"func": "is_boolean", "values": ["true"], "expected": True}, - {"func": "is_boolean", "values": ["false"], "expected": True}, - {"func": "is_number", "values": ["1"], "expected": True}, - {"func": "is_number", "values": ["1.1"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, - {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, - {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, - {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, - {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, - {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, - {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, - {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, - { - "func": "get_host_path", - "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], - "expected": "/mnt/test", - }, - { - "func": "get_host_path", - "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], - "expected": "/mnt/test123", - }, - ] - - for test in tests: - print(test["func"], test) - func = render.funcs[test["func"]] - if test.get("expect_raise", False): - with pytest.raises(Exception): - func(*test["values"]) - elif test.get("expect_regex"): - r = func(*test["values"]) - assert re.match(test["expect_regex"], r) is not None - else: - r = func(*test["values"]) - assert r == test["expected"] diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_healthcheck.py b/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_healthcheck.py deleted file mode 100644 index 8267b986b4..0000000000 --- a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_healthcheck.py +++ /dev/null @@ -1,187 +0,0 @@ -import pytest - -from render import Render - - -@pytest.fixture -def mock_values(): - return { - "images": { - "test_image": { - "repository": "nginx", - "tag": "latest", - } - }, - } - - -def test_disable_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == {"disable": True} - - -def test_set_custom_test(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test("echo $1") - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": "echo $$1", - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_custom_test_array(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "10s", - "timeout": "5s", - "retries": 30, - "start_period": "10s", - } - - -def test_set_options(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) - c1.healthcheck.set_interval(9) - c1.healthcheck.set_timeout(8) - c1.healthcheck.set_retries(7) - c1.healthcheck.set_start_period(6) - output = render.render() - assert output["services"]["test_container"]["healthcheck"] == { - "test": ["CMD", "echo", "$$1"], - "interval": "9s", - "timeout": "8s", - "retries": 7, - "start_period": "6s", - } - - -def test_adding_test_when_disabled(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.disable() - with pytest.raises(Exception): - c1.healthcheck.set_custom_test("echo $1") - - -def test_not_adding_test(mock_values): - render = Render(mock_values) - render.add_container("test_container", "test_image") - with pytest.raises(Exception): - render.render() - - -def test_invalid_path(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - with pytest.raises(Exception): - c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) - - -def test_http_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("http", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep -q "200 OK"'""" # noqa - ) - - -def test_curl_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" - ) - - -def test_curl_healthcheck_with_headers(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' - ) - - -def test_wget_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "wget --spider --quiet http://127.0.0.1:8080/health" - ) - - -def test_netcat_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("netcat", {"port": 8080}) - output = render.render() - assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" - - -def test_tcp_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("tcp", {"port": 8080}) - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" - ) - - -def test_redis_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("redis") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" - ) - - -def test_postgres_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("postgres") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" - ) - - -def test_mariadb_healthcheck(mock_values): - render = Render(mock_values) - c1 = render.add_container("test_container", "test_image") - c1.healthcheck.set_test("mariadb") - output = render.render() - assert ( - output["services"]["test_container"]["healthcheck"]["test"] - == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" - ) diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/validations.py b/ix-dev/community/zerotier/templates/library/base_v2_0_25/validations.py deleted file mode 100644 index 155c2e1091..0000000000 --- a/ix-dev/community/zerotier/templates/library/base_v2_0_25/validations.py +++ /dev/null @@ -1,227 +0,0 @@ -import re -import ipaddress - -try: - from .error import RenderError -except ImportError: - from error import RenderError - -OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") - - -def valid_sysctl_or_raise(sysctl: str, host_network: bool): - if not sysctl: - raise RenderError("Sysctl cannot be empty") - if host_network and sysctl.startswith("net."): - raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") - - valid_sysctls = [ - "kernel.msgmax", - "kernel.msgmnb", - "kernel.msgmni", - "kernel.sem", - "kernel.shmall", - "kernel.shmmax", - "kernel.shmmni", - "kernel.shm_rmid_forced", - ] - # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls - if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: - raise RenderError( - f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" - ) - return sysctl - - -def valid_redis_password_or_raise(password: str): - forbidden_chars = [" ", "'"] - for char in forbidden_chars: - if char in password: - raise RenderError(f"Redis password cannot contain [{char}]") - - -def valid_octal_mode_or_raise(mode: str): - mode = str(mode) - if not OCTAL_MODE_REGEX.match(mode): - raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") - return mode - - -def valid_host_path_propagation(propagation: str): - valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") - if propagation not in valid_propagations: - raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") - return propagation - - -def valid_portal_scheme_or_raise(scheme: str): - schemes = ("http", "https") - if scheme not in schemes: - raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") - return scheme - - -def valid_port_or_raise(port: int): - if port < 1 or port > 65535: - raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") - return port - - -def valid_ip_or_raise(ip: str): - try: - ipaddress.ip_address(ip) - except ValueError: - raise RenderError(f"Invalid IP address [{ip}]") - return ip - - -def valid_port_mode_or_raise(mode: str): - modes = ("ingress", "host") - if mode not in modes: - raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") - return mode - - -def valid_port_protocol_or_raise(protocol: str): - protocols = ("tcp", "udp") - if protocol not in protocols: - raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") - return protocol - - -def valid_depend_condition_or_raise(condition: str): - valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") - if condition not in valid_conditions: - raise RenderError( - f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" - ) - return condition - - -def valid_cgroup_perm_or_raise(cgroup_perm: str): - valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") - if cgroup_perm not in valid_cgroup_perms: - raise RenderError( - f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" - ) - return cgroup_perm - - -def allowed_dns_opt_or_raise(dns_opt: str): - disallowed_dns_opts = [] - if dns_opt in disallowed_dns_opts: - raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") - return dns_opt - - -def valid_http_path_or_raise(path: str): - path = _valid_path_or_raise(path) - return path - - -def valid_fs_path_or_raise(path: str): - # There is no reason to allow / as a path, - # either on host or in a container side. - if path == "/": - raise RenderError(f"Path [{path}] cannot be [/]") - path = _valid_path_or_raise(path) - return path - - -def _valid_path_or_raise(path: str): - if path == "": - raise RenderError(f"Path [{path}] cannot be empty") - if not path.startswith("/"): - raise RenderError(f"Path [{path}] must start with /") - if "//" in path: - raise RenderError(f"Path [{path}] cannot contain [//]") - return path - - -def allowed_device_or_raise(path: str): - disallowed_devices = ["/dev/dri", "/dev/bus/usb"] - if path in disallowed_devices: - raise RenderError(f"Device [{path}] is not allowed to be manually added.") - return path - - -def valid_network_mode_or_raise(mode: str, containers: list[str]): - valid_modes = ("host", "none") - if mode in valid_modes: - return mode - - if mode.startswith("service:"): - if mode[8:] not in containers: - raise RenderError(f"Service [{mode[8:]}] not found") - return mode - - raise RenderError( - f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" - ) - - -def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): - valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") - if policy not in valid_restart_policies: - raise RenderError( - f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" - ) - if policy != "on-failure" and maximum_retry_count != 0: - raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") - - if maximum_retry_count < 0: - raise RenderError("Maximum retry count must be a positive integer") - - return policy - - -def valid_cap_or_raise(cap: str): - valid_policies = ( - "ALL", - "AUDIT_CONTROL", - "AUDIT_READ", - "AUDIT_WRITE", - "BLOCK_SUSPEND", - "BPF", - "CHECKPOINT_RESTORE", - "CHOWN", - "DAC_OVERRIDE", - "DAC_READ_SEARCH", - "FOWNER", - "FSETID", - "IPC_LOCK", - "IPC_OWNER", - "KILL", - "LEASE", - "LINUX_IMMUTABLE", - "MAC_ADMIN", - "MAC_OVERRIDE", - "MKNOD", - "NET_ADMIN", - "NET_BIND_SERVICE", - "NET_BROADCAST", - "NET_RAW", - "PERFMON", - "SETFCAP", - "SETGID", - "SETPCAP", - "SETUID", - "SYS_ADMIN", - "SYS_BOOT", - "SYS_CHROOT", - "SYS_MODULE", - "SYS_NICE", - "SYS_PACCT", - "SYS_PTRACE", - "SYS_RAWIO", - "SYS_RESOURCE", - "SYS_TIME", - "SYS_TTY_CONFIG", - "SYSLOG", - "WAKE_ALARM", - ) - - if cap not in valid_policies: - raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") - - return cap diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/__init__.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/__init__.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/__init__.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/__init__.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/configs.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/configs.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/configs.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/configs.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/container.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/container.py new file mode 100644 index 0000000000..701f64bfeb --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/container.py @@ -0,0 +1,339 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .configs import ContainerConfigs + from .depends import Depends + from .deploy import Deploy + from .devices import Devices + from .dns import Dns + from .environment import Environment + from .error import RenderError + from .formatter import escape_dollar, get_image_with_hashed_data + from .healthcheck import Healthcheck + from .labels import Labels + from .ports import Ports + from .restart import RestartPolicy + from .validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from .storage import Storage + from .sysctls import Sysctls +except ImportError: + from configs import ContainerConfigs + from depends import Depends + from deploy import Deploy + from devices import Devices + from dns import Dns + from environment import Environment + from error import RenderError + from formatter import escape_dollar, get_image_with_hashed_data + from healthcheck import Healthcheck + from labels import Labels + from ports import Ports + from restart import RestartPolicy + from validations import valid_network_mode_or_raise, valid_cap_or_raise, valid_pull_policy_or_raise + from storage import Storage + from sysctls import Sysctls + + +class Container: + def __init__(self, render_instance: "Render", name: str, image: str): + self._render_instance = render_instance + + self._name: str = name + self._image: str = self._resolve_image(image) + self._build_image: str = "" + self._pull_policy: str = "" + self._user: str = "" + self._tty: bool = False + self._stdin_open: bool = False + self._init: bool | None = None + self._read_only: bool | None = None + self._hostname: str = "" + self._cap_drop: set[str] = set(["ALL"]) # Drop all capabilities by default and add caps granularly + self._cap_add: set[str] = set() + self._security_opt: set[str] = set(["no-new-privileges"]) + self._privileged: bool = False + self._group_add: set[int | str] = set() + self._network_mode: str = "" + self._entrypoint: list[str] = [] + self._command: list[str] = [] + self._grace_period: int | None = None + self._shm_size: int | None = None + self._storage: Storage = Storage(self._render_instance) + self.sysctls: Sysctls = Sysctls(self._render_instance, self) + self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs) + self.deploy: Deploy = Deploy(self._render_instance) + self.networks: set[str] = set() + self.devices: Devices = Devices(self._render_instance) + self.environment: Environment = Environment(self._render_instance, self.deploy.resources) + self.dns: Dns = Dns(self._render_instance) + self.depends: Depends = Depends(self._render_instance) + self.healthcheck: Healthcheck = Healthcheck(self._render_instance) + self.labels: Labels = Labels(self._render_instance) + self.restart: RestartPolicy = RestartPolicy(self._render_instance) + self.ports: Ports = Ports(self._render_instance) + + self._auto_set_network_mode() + self._auto_add_labels() + self._auto_add_groups() + + def _auto_add_groups(self): + self.add_group(568) + + def _auto_set_network_mode(self): + if self._render_instance.values.get("network", {}).get("host_network", False): + self.set_network_mode("host") + + def _auto_add_labels(self): + labels = self._render_instance.values.get("labels", []) + if not labels: + return + + for label in labels: + containers = label.get("containers", []) + if not containers: + raise RenderError(f'Label [{label.get("key", "")}] must have at least one container') + + if self._name in containers: + self.labels.add_label(label["key"], label["value"]) + + def _resolve_image(self, image: str): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError( + f"Image [{image}] not found in values. " f"Available images: [{', '.join(images.keys())}]" + ) + repo = images[image].get("repository", "") + tag = images[image].get("tag", "") + + if not repo: + raise RenderError(f"Repository not found for image [{image}]") + if not tag: + raise RenderError(f"Tag not found for image [{image}]") + + return f"{repo}:{tag}" + + def build_image(self, content: list[str | None]): + dockerfile = f"FROM {self._image}\n" + for line in content: + if not line: + continue + if line.startswith("FROM"): + # TODO: This will also block multi-stage builds + # We can revisit this later if we need it + raise RenderError( + "FROM cannot be used in build image. Define the base image when creating the container." + ) + dockerfile += line + "\n" + + self._build_image = dockerfile + self._image = get_image_with_hashed_data(self._image, dockerfile) + + def set_pull_policy(self, pull_policy: str): + self._pull_policy = valid_pull_policy_or_raise(pull_policy) + + def set_user(self, user: int, group: int): + for i in (user, group): + if not isinstance(i, int) or i < 0: + raise RenderError(f"User/Group [{i}] is not valid") + self._user = f"{user}:{group}" + + def add_group(self, group: int | str): + if isinstance(group, str): + group = str(group).strip() + if group.isdigit(): + raise RenderError(f"Group is a number [{group}] but passed as a string") + + if group in self._group_add: + raise RenderError(f"Group [{group}] already added") + self._group_add.add(group) + + def get_current_groups(self) -> list[str]: + return [str(g) for g in self._group_add] + + def set_tty(self, enabled: bool = False): + self._tty = enabled + + def set_stdin(self, enabled: bool = False): + self._stdin_open = enabled + + def set_init(self, enabled: bool = False): + self._init = enabled + + def set_read_only(self, enabled: bool = False): + self._read_only = enabled + + def set_hostname(self, hostname: str): + self._hostname = hostname + + def set_grace_period(self, grace_period: int): + if grace_period < 0: + raise RenderError(f"Grace period [{grace_period}] cannot be negative") + self._grace_period = grace_period + + def set_privileged(self, enabled: bool = False): + self._privileged = enabled + + def clear_caps(self): + self._cap_add.clear() + self._cap_drop.clear() + + def add_caps(self, caps: list[str]): + for c in caps: + if c in self._cap_add: + raise RenderError(f"Capability [{c}] already added") + self._cap_add.add(valid_cap_or_raise(c)) + + def add_security_opt(self, opt: str): + if opt in self._security_opt: + raise RenderError(f"Security Option [{opt}] already added") + self._security_opt.add(opt) + + def remove_security_opt(self, opt: str): + self._security_opt.remove(opt) + + def set_network_mode(self, mode: str): + self._network_mode = valid_network_mode_or_raise(mode, self._render_instance.container_names()) + + def set_entrypoint(self, entrypoint: list[str]): + self._entrypoint = [escape_dollar(str(e)) for e in entrypoint] + + def set_command(self, command: list[str]): + self._command = [escape_dollar(str(e)) for e in command] + + def add_storage(self, mount_path: str, config: "IxStorage"): + self._storage.add(mount_path, config) + + def add_docker_socket(self, read_only: bool = True, mount_path: str = "/var/run/docker.sock"): + self.add_group(999) + self._storage._add_docker_socket(read_only, mount_path) + + def add_tun_device(self, read_only: bool = True, mount_path: str = "/dev/net/tun"): + self._storage._add_tun_device(read_only, mount_path) + + def add_snd_device(self): + self.add_group(29) + self.devices._add_snd_device() + + def set_shm_size_mb(self, size: int): + self._shm_size = size + + # Easily remove devices from the container + # Useful in dependencies like postgres and redis + # where there is no need to pass devices to them + def remove_devices(self): + self.deploy.resources.remove_devices() + self.devices.remove_devices() + + @property + def storage(self): + return self._storage + + def render(self) -> dict[str, Any]: + if self._network_mode and self.networks: + raise RenderError("Cannot set both [network_mode] and [networks]") + + result = { + "image": self._image, + "platform": "linux/amd64", + "tty": self._tty, + "stdin_open": self._stdin_open, + "restart": self.restart.render(), + } + + if self._pull_policy: + result["pull_policy"] = self._pull_policy + + if self.healthcheck.has_healthcheck(): + result["healthcheck"] = self.healthcheck.render() + + if self._hostname: + result["hostname"] = self._hostname + + if self._build_image: + result["build"] = {"tags": [self._image], "dockerfile_inline": self._build_image} + + if self.configs.has_configs(): + result["configs"] = self.configs.render() + + if self._init is not None: + result["init"] = self._init + + if self._read_only is not None: + result["read_only"] = self._read_only + + if self._grace_period is not None: + result["stop_grace_period"] = f"{self._grace_period}s" + + if self._user: + result["user"] = self._user + + if self.deploy.resources.has_gpus() or self.devices.has_gpus(): + self.add_group(44) # video + self.add_group(107) # render + + if self._group_add: + result["group_add"] = sorted(self._group_add, key=lambda g: (isinstance(g, str), g)) + + if self._shm_size is not None: + result["shm_size"] = f"{self._shm_size}M" + + if self._privileged is not None: + result["privileged"] = self._privileged + + if self._cap_drop: + result["cap_drop"] = sorted(self._cap_drop) + + if self._cap_add: + result["cap_add"] = sorted(self._cap_add) + + if self._security_opt: + result["security_opt"] = sorted(self._security_opt) + + if self._network_mode: + result["network_mode"] = self._network_mode + + if self.sysctls.has_sysctls(): + result["sysctls"] = self.sysctls.render() + + if self._network_mode != "host": + if self.ports.has_ports(): + result["ports"] = self.ports.render() + + if self._entrypoint: + result["entrypoint"] = self._entrypoint + + if self._command: + result["command"] = self._command + + if self.devices.has_devices(): + result["devices"] = self.devices.render() + + if self.deploy.has_deploy(): + result["deploy"] = self.deploy.render() + + if self.environment.has_variables(): + result["environment"] = self.environment.render() + + if self.labels.has_labels(): + result["labels"] = self.labels.render() + + if self.dns.has_dns_nameservers(): + result["dns"] = self.dns.render_dns_nameservers() + + if self.dns.has_dns_searches(): + result["dns_search"] = self.dns.render_dns_searches() + + if self.dns.has_dns_opts(): + result["dns_opt"] = self.dns.render_dns_opts() + + if self.depends.has_dependencies(): + result["depends_on"] = self.depends.render() + + if self._storage.has_mounts(): + result["volumes"] = self._storage.render() + + return result diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/depends.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/depends.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/depends.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/depends.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/deploy.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/deploy.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/deploy.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/deploy.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/deps.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/deps.py new file mode 100644 index 0000000000..e96849f979 --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/deps.py @@ -0,0 +1,32 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .deps_postgres import PostgresContainer, PostgresConfig + from .deps_redis import RedisContainer, RedisConfig + from .deps_mariadb import MariadbContainer, MariadbConfig + from .deps_perms import PermsContainer +except ImportError: + from deps_postgres import PostgresContainer, PostgresConfig + from deps_redis import RedisContainer, RedisConfig + from deps_mariadb import MariadbContainer, MariadbConfig + from deps_perms import PermsContainer + + +class Deps: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def perms(self, name: str): + return PermsContainer(self._render_instance, name) + + def postgres(self, name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer): + return PostgresContainer(self._render_instance, name, image, config, perms_instance) + + def redis(self, name: str, image: str, config: RedisConfig, perms_instance: PermsContainer): + return RedisContainer(self._render_instance, name, image, config, perms_instance) + + def mariadb(self, name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer): + return MariadbContainer(self._render_instance, name, image, config, perms_instance) diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/deps_mariadb.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/deps_mariadb.py new file mode 100644 index 0000000000..baf863b370 --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/deps_mariadb.py @@ -0,0 +1,65 @@ +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class MariadbConfig(TypedDict): + user: str + password: str + database: str + root_password: NotRequired[str] + port: NotRequired[int] + auto_upgrade: NotRequired[bool] + volume: "IxStorage" + + +class MariadbContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: MariadbConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for mariadb") + + port = valid_port_or_raise(config.get("port") or 3306) + root_password = config.get("root_password") or config["password"] + auto_upgrade = config.get("auto_upgrade", True) + + c = self._render_instance.add_container(name, image) + c.set_user(999, 999) + c.healthcheck.set_test("mariadb") + c.remove_devices() + + c.add_storage("/var/lib/mysql", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_mariadb_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + c.environment.add_env("MARIADB_USER", config["user"]) + c.environment.add_env("MARIADB_PASSWORD", config["password"]) + c.environment.add_env("MARIADB_ROOT_PASSWORD", root_password) + c.environment.add_env("MARIADB_DATABASE", config["database"]) + c.environment.add_env("MARIADB_AUTO_UPGRADE", str(auto_upgrade).lower()) + c.set_command(["--port", str(port)]) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/deps_perms.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/deps_perms.py new file mode 100644 index 0000000000..cdc5a3820a --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/deps_perms.py @@ -0,0 +1,252 @@ +import json +import pathlib +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .validations import valid_octal_mode_or_raise, valid_fs_path_or_raise +except ImportError: + from error import RenderError + from validations import valid_octal_mode_or_raise, valid_fs_path_or_raise + + +class PermsContainer: + def __init__(self, render_instance: "Render", name: str): + self._render_instance = render_instance + self._name = name + self.actions: set[str] = set() + self.parsed_configs: list[dict] = [] + + def add_or_skip_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + identifier = self.normalize_identifier_for_path(identifier) + if identifier in self.actions: + raise RenderError(f"Action with id [{identifier}] already used for another permission action") + + parsed_action = self.parse_action(identifier, volume_config, action_config) + if parsed_action: + self.parsed_configs.append(parsed_action) + self.actions.add(identifier) + + def parse_action(self, identifier: str, volume_config: "IxStorage", action_config: dict): + valid_modes = [ + "always", # Always set permissions, without checking. + "check", # Checks if permissions are correct, and set them if not. + ] + mode = action_config.get("mode", "check") + uid = action_config.get("uid", None) + gid = action_config.get("gid", None) + chmod = action_config.get("chmod", None) + recursive = action_config.get("recursive", False) + mount_path = pathlib.Path("/mnt/permission", identifier).as_posix() + is_temporary = False + + vol_type = volume_config.get("type", "") + match vol_type: + case "temporary": + # If it is a temporary volume, we force auto permissions + # and set is_temporary to True, so it will be cleaned up + is_temporary = True + recursive = True + case "volume": + if not volume_config.get("volume_config", {}).get("auto_permissions", False): + return None + case "host_path": + host_path_config = volume_config.get("host_path_config", {}) + # Skip when ACL enabled + if host_path_config.get("acl_enable", False): + return None + if not host_path_config.get("auto_permissions", False): + return None + case "ix_volume": + ix_vol_config = volume_config.get("ix_volume_config", {}) + # Skip when ACL enabled + if ix_vol_config.get("acl_enable", False): + return None + # For ix_volumes, we default to auto_permissions = True + if not ix_vol_config.get("auto_permissions", True): + return None + case _: + # Skip for other types + return None + + if mode not in valid_modes: + raise RenderError(f"Expected [mode] to be one of [{', '.join(valid_modes)}], got [{mode}]") + if not isinstance(uid, int) or not isinstance(gid, int): + raise RenderError("Expected [uid] and [gid] to be set when [auto_permissions] is enabled") + if chmod is not None: + chmod = valid_octal_mode_or_raise(chmod) + + mount_path = valid_fs_path_or_raise(mount_path) + return { + "mount_path": mount_path, + "volume_config": volume_config, + "action_data": { + "mount_path": mount_path, + "is_temporary": is_temporary, + "identifier": identifier, + "recursive": recursive, + "mode": mode, + "uid": uid, + "gid": gid, + "chmod": chmod, + }, + } + + def normalize_identifier_for_path(self, identifier: str): + return identifier.rstrip("/").lstrip("/").lower().replace("/", "_").replace(".", "-").replace(" ", "-") + + def has_actions(self): + return bool(self.actions) + + def activate(self): + if len(self.parsed_configs) != len(self.actions): + raise RenderError("Number of actions and parsed configs does not match") + + if not self.has_actions(): + raise RenderError("No actions added. Check if there are actions before activating") + + # Add the container and set it up + c = self._render_instance.add_container(self._name, "python_permissions_image") + c.set_user(0, 0) + c.add_caps(["CHOWN", "FOWNER", "DAC_OVERRIDE"]) + c.set_network_mode("none") + + # Don't attach any devices + c.remove_devices() + + c.deploy.resources.set_profile("medium") + c.restart.set_policy("on-failure", maximum_retry_count=1) + c.healthcheck.disable() + + c.set_entrypoint(["python3", "/script/run.py"]) + script = "#!/usr/bin/env python3\n" + script += get_script() + c.configs.add("permissions_run_script", script, "/script/run.py", "0700") + + actions_data: list[dict] = [] + for parsed in self.parsed_configs: + c.add_storage(parsed["mount_path"], parsed["volume_config"]) + actions_data.append(parsed["action_data"]) + + actions_data_json = json.dumps(actions_data) + c.configs.add("permissions_actions_data", actions_data_json, "/script/actions.json", "0500") + + +def get_script(): + return """ +import os +import json +import time +import shutil + +with open("/script/actions.json", "r") as f: + actions_data = json.load(f) + +if not actions_data: + # If this script is called, there should be actions data + raise ValueError("No actions data found") + +def fix_perms(path, chmod, recursive=False): + print(f"Changing permissions{' recursively ' if recursive else ' '}to {chmod} on: [{path}]") + os.chmod(path, int(chmod, 8)) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chmod(os.path.join(root, f), int(chmod, 8)) + print("Permissions after changes:") + print_chmod_stat() + +def fix_owner(path, uid, gid, recursive=False): + print(f"Changing ownership{' recursively ' if recursive else ' '}to {uid}:{gid} on: [{path}]") + os.chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for f in files: + os.chown(os.path.join(root, f), uid, gid) + print("Ownership after changes:") + print_chown_stat() + +def print_chown_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Ownership: [{curr_stat.st_uid}:{curr_stat.st_gid}]") + +def print_chmod_stat(): + curr_stat = os.stat(action["mount_path"]) + print(f"Permissions: [{oct(curr_stat.st_mode)[3:]}]") + +def print_chown_diff(curr_stat, uid, gid): + print(f"Ownership: wanted [{uid}:{gid}], got [{curr_stat.st_uid}:{curr_stat.st_gid}].") + +def print_chmod_diff(curr_stat, mode): + print(f"Permissions: wanted [{mode}], got [{oct(curr_stat.st_mode)[3:]}].") + +def perform_action(action): + start_time = time.time() + print(f"=== Applying configuration on volume with identifier [{action['identifier']}] ===") + + if not os.path.isdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not a directory, skipping...") + return + + if action["is_temporary"]: + print(f"Path [{action['mount_path']}] is a temporary directory, ensuring it is empty...") + for item in os.listdir(action["mount_path"]): + item_path = os.path.join(action["mount_path"], item) + + # Exclude the safe directory, where we can use to mount files temporarily + if os.path.basename(item_path) == "ix-safe": + continue + if os.path.isdir(item_path): + shutil.rmtree(item_path) + else: + os.remove(item_path) + + if not action["is_temporary"] and os.listdir(action["mount_path"]): + print(f"Path [{action['mount_path']}] is not empty, skipping...") + return + + print(f"Current Ownership and Permissions on [{action['mount_path']}]:") + curr_stat = os.stat(action["mount_path"]) + print_chown_diff(curr_stat, action["uid"], action["gid"]) + print_chmod_diff(curr_stat, action["chmod"]) + print("---") + + if action["mode"] == "always": + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + return + + elif action["mode"] == "check": + if curr_stat.st_uid != action["uid"] or curr_stat.st_gid != action["gid"]: + print("Ownership is incorrect. Fixing...") + fix_owner(action["mount_path"], action["uid"], action["gid"], action["recursive"]) + else: + print("Ownership is correct. Skipping...") + + if not action["chmod"]: + print("Skipping permissions check, chmod is falsy") + else: + if oct(curr_stat.st_mode)[3:] != action["chmod"]: + print("Permissions are incorrect. Fixing...") + fix_perms(action["mount_path"], action["chmod"], action["recursive"]) + else: + print("Permissions are correct. Skipping...") + + print(f"Time taken: {(time.time() - start_time) * 1000:.2f}ms") + print(f"=== Finished applying configuration on volume with identifier [{action['identifier']}] ==") + print() + +if __name__ == "__main__": + start_time = time.time() + for action in actions_data: + perform_action(action) + print(f"Total time taken: {(time.time() - start_time) * 1000:.2f}ms") +""" diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/deps_postgres.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/deps_postgres.py new file mode 100644 index 0000000000..c40e900d07 --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/deps_postgres.py @@ -0,0 +1,279 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise + + +class PostgresConfig(TypedDict): + user: str + password: str + database: str + port: NotRequired[int] + volume: "IxStorage" + + +MAX_POSTGRES_VERSION = 17 + + +class PostgresContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: PostgresConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + self._data_dir = "/var/lib/postgresql/data" + self._upgrade_name = f"{self._name}_upgrade" + self._upgrade_container = None + + for key in ("user", "password", "database", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for postgres") + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + + c.set_user(999, 999) + c.healthcheck.set_test("postgres") + c.remove_devices() + c.add_storage(self._data_dir, config["volume"]) + + common_variables = { + "POSTGRES_USER": config["user"], + "POSTGRES_PASSWORD": config["password"], + "POSTGRES_DB": config["database"], + "POSTGRES_PORT": port, + } + + for k, v in common_variables.items(): + c.environment.add_env(k, v) + + perms_instance.add_or_skip_action( + f"{self._name}_postgres_data", config["volume"], {"uid": 999, "gid": 999, "mode": "check"} + ) + + repo = self._get_repo(image) + # eg we don't want to handle upgrades of pg_vector at the moment + if repo == "postgres": + target_major_version = self._get_target_version(image) + upg = self._render_instance.add_container(self._upgrade_name, image) + upg.build_image(get_build_manifest()) + upg.set_entrypoint(["/bin/bash", "-c", "/upgrade.sh"]) + upg.configs.add("pg_container_upgrade.sh", get_upgrade_script(), "/upgrade.sh", "0755") + upg.restart.set_policy("on-failure", 1) + upg.set_user(999, 999) + upg.healthcheck.disable() + upg.remove_devices() + upg.add_storage(self._data_dir, config["volume"]) + for k, v in common_variables.items(): + upg.environment.add_env(k, v) + + upg.environment.add_env("TARGET_VERSION", target_major_version) + upg.environment.add_env("DATA_DIR", self._data_dir) + + self._upgrade_container = upg + + c.depends.add_dependency(self._upgrade_name, "service_completed_successfully") + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + @property + def container(self): + return self._container + + def add_dependency(self, container_name: str, condition: str): + self._container.depends.add_dependency(container_name, condition) + if self._upgrade_container: + self._upgrade_container.depends.add_dependency(container_name, condition) + + def _get_port(self): + return self._config.get("port") or 5432 + + def _get_repo(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + repo = images[image].get("repository", "") + if not repo: + raise RenderError("Could not determine repo") + return repo + + def _get_target_version(self, image): + images = self._render_instance.values["images"] + if image not in images: + raise RenderError(f"Image [{image}] not found in values. Available images: [{', '.join(images.keys())}]") + tag = images[image].get("tag", "") + tag = str(tag) # Account for tags like 16.6 + target_major_version = tag.split(".")[0] + + try: + target_major_version = int(target_major_version) + except ValueError: + raise RenderError(f"Could not determine target major version from tag [{tag}]") + + if target_major_version > MAX_POSTGRES_VERSION: + raise RenderError(f"Postgres version [{target_major_version}] is not supported") + + return target_major_version + + def get_url(self, variant: str): + user = urllib.parse.quote_plus(self._config["user"]) + password = urllib.parse.quote_plus(self._config["password"]) + creds = f"{user}:{password}" + addr = f"{self._name}:{self._get_port()}" + db = self._config["database"] + + match variant: + case "postgres": + return f"postgres://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql": + return f"postgresql://{creds}@{addr}/{db}?sslmode=disable" + case "postgresql_no_creds": + return f"postgresql://{addr}/{db}?sslmode=disable" + case "host_port": + return addr + case _: + raise RenderError(f"Expected [variant] to be one of [postgres, postgresql], got [{variant}]") + + +def get_build_manifest() -> list[str | None]: + return [ + f"RUN apt-get update && apt-get install -y {' '.join(get_upgrade_packages())}", + "WORKDIR /tmp", + ] + + +def get_upgrade_packages(): + return [ + "rsync", + "postgresql-13", + "postgresql-14", + "postgresql-15", + "postgresql-16", + ] + + +def get_upgrade_script(): + return """ +#!/bin/bash +set -euo pipefail + +get_bin_path() { + local version=$1 + echo "/usr/lib/postgresql/$version/bin" +} + +log() { + echo "[ix-postgres-upgrade] - [$(date +'%Y-%m-%d %H:%M:%S')] - $1" +} + +check_writable() { + local path=$1 + if [ ! -w "$path" ]; then + log "$path is not writable" + exit 1 + fi +} + +check_writable "$DATA_DIR" + +# Don't do anything if its a fresh install. +if [ ! -f "$DATA_DIR/PG_VERSION" ]; then + log "File $DATA_DIR/PG_VERSION does not exist. Assuming this is a fresh install." + exit 0 +fi + +# Don't do anything if we're already at the target version. +OLD_VERSION=$(cat "$DATA_DIR/PG_VERSION") +log "Current version: $OLD_VERSION" +log "Target version: $TARGET_VERSION" +if [ "$OLD_VERSION" -eq "$TARGET_VERSION" ]; then + log "Already at target version $TARGET_VERSION" + exit 0 +fi + +# Fail if we're downgrading. +if [ "$OLD_VERSION" -gt "$TARGET_VERSION" ]; then + log "Cannot downgrade from $OLD_VERSION to $TARGET_VERSION" + exit 1 +fi + +export OLD_PG_BINARY=$(get_bin_path "$OLD_VERSION") +if [ ! -f "$OLD_PG_BINARY/pg_upgrade" ]; then + log "File $OLD_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_PG_BINARY=$(get_bin_path "$TARGET_VERSION") +if [ ! -f "$NEW_PG_BINARY/pg_upgrade" ]; then + log "File $NEW_PG_BINARY/pg_upgrade does not exist." + exit 1 +fi + +export NEW_DATA_DIR="/tmp/new-data-dir" +if [ -d "$NEW_DATA_DIR" ]; then + log "Directory $NEW_DATA_DIR already exists." + exit 1 +fi + +export PGUSER="$POSTGRES_USER" +log "Creating new data dir and initializing..." +PGDATA="$NEW_DATA_DIR" eval "initdb --username=$POSTGRES_USER --pwfile=<(echo $POSTGRES_PASSWORD)" + +timestamp=$(date +%Y%m%d%H%M%S) +backup_name="backup-$timestamp-$OLD_VERSION-$TARGET_VERSION.tar.gz" +log "Backing up $DATA_DIR to $NEW_DATA_DIR/$backup_name" +tar -czf "$NEW_DATA_DIR/$backup_name" "$DATA_DIR" + +log "Using old pg_upgrade [$OLD_PG_BINARY/pg_upgrade]" +log "Using new pg_upgrade [$NEW_PG_BINARY/pg_upgrade]" +log "Checking upgrade compatibility of $OLD_VERSION to $TARGET_VERSION..." + +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql \ + --check + +log "Compatibility check passed." + +log "Upgrading from $OLD_VERSION to $TARGET_VERSION..." +"$NEW_PG_BINARY"/pg_upgrade \ + --old-bindir="$OLD_PG_BINARY" \ + --new-bindir="$NEW_PG_BINARY" \ + --old-datadir="$DATA_DIR" \ + --new-datadir="$NEW_DATA_DIR" \ + --socketdir /var/run/postgresql + +log "Upgrade complete." + +log "Copying old pg_hba.conf to new pg_hba.conf" +# We need to carry this over otherwise +cp "$DATA_DIR/pg_hba.conf" "$NEW_DATA_DIR/pg_hba.conf" + +log "Replacing contents of $DATA_DIR with contents of $NEW_DATA_DIR (including the backup)." +rsync --archive --delete "$NEW_DATA_DIR/" "$DATA_DIR/" + +log "Removing $NEW_DATA_DIR." +rm -rf "$NEW_DATA_DIR" + +log "Done." +""" diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/deps_redis.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/deps_redis.py new file mode 100644 index 0000000000..d49ebe7683 --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/deps_redis.py @@ -0,0 +1,71 @@ +import urllib.parse +from typing import TYPE_CHECKING, TypedDict, NotRequired + +if TYPE_CHECKING: + from render import Render + from storage import IxStorage + +try: + from .error import RenderError + from .deps_perms import PermsContainer + from .validations import valid_port_or_raise, valid_redis_password_or_raise +except ImportError: + from error import RenderError + from deps_perms import PermsContainer + from validations import valid_port_or_raise, valid_redis_password_or_raise + + +class RedisConfig(TypedDict): + password: str + port: NotRequired[int] + volume: "IxStorage" + + +class RedisContainer: + def __init__( + self, render_instance: "Render", name: str, image: str, config: RedisConfig, perms_instance: PermsContainer + ): + self._render_instance = render_instance + self._name = name + self._config = config + + for key in ("password", "volume"): + if key not in config: + raise RenderError(f"Expected [{key}] to be set for redis") + + valid_redis_password_or_raise(config["password"]) + + port = valid_port_or_raise(self._get_port()) + + c = self._render_instance.add_container(name, image) + c.set_user(1001, 0) + c.healthcheck.set_test("redis") + c.remove_devices() + + c.add_storage("/bitnami/redis/data", config["volume"]) + perms_instance.add_or_skip_action( + f"{self._name}_redis_data", config["volume"], {"uid": 1001, "gid": 0, "mode": "check"} + ) + + c.environment.add_env("ALLOW_EMPTY_PASSWORD", "no") + c.environment.add_env("REDIS_PASSWORD", config["password"]) + c.environment.add_env("REDIS_PORT_NUMBER", port) + + # Store container for further configuration + # For example: c.depends.add_dependency("other_container", "service_started") + self._container = c + + def _get_port(self): + return self._config.get("port") or 6379 + + def get_url(self, variant: str): + addr = f"{self._name}:{self._get_port()}" + password = urllib.parse.quote_plus(self._config["password"]) + + match variant: + case "redis": + return f"redis://default:{password}@{addr}" + + @property + def container(self): + return self._container diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/device.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/device.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/device.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/device.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/devices.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/devices.py new file mode 100644 index 0000000000..b6139371ee --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/devices.py @@ -0,0 +1,68 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .device import Device +except ImportError: + from error import RenderError + from device import Device + + +class Devices: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._devices: set[Device] = set() + + # Tracks all container device paths to make sure they are not duplicated + self._container_device_paths: set[str] = set() + # Scan values for devices we should automatically add + # for example /dev/dri for gpus + self._auto_add_devices_from_values() + + def _auto_add_devices_from_values(self): + resources = self._render_instance.values.get("resources", {}) + + if resources.get("gpus", {}).get("use_all_gpus", False): + self.add_device("/dev/dri", "/dev/dri", allow_disallowed=True) + if resources["gpus"].get("kfd_device_exists", False): + self.add_device("/dev/kfd", "/dev/kfd", allow_disallowed=True) # AMD ROCm + + def add_device(self, host_device: str, container_device: str, cgroup_perm: str = "", allow_disallowed=False): + # Host device can be mapped to multiple container devices, + # so we only make sure container devices are not duplicated + if container_device in self._container_device_paths: + raise RenderError(f"Device with container path [{container_device}] already added") + + self._devices.add(Device(host_device, container_device, cgroup_perm, allow_disallowed)) + self._container_device_paths.add(container_device) + + def add_usb_bus(self): + self.add_device("/dev/bus/usb", "/dev/bus/usb", allow_disallowed=True) + + def _add_snd_device(self): + self.add_device("/dev/snd", "/dev/snd", allow_disallowed=True) + + def has_devices(self): + return len(self._devices) > 0 + + # Mainly will be used from dependencies + # There is no reason to pass devices to + # redis or postgres for example + def remove_devices(self): + self._devices.clear() + self._container_device_paths.clear() + + # Check if there are any gpu devices + # Used to determine if we should add groups + # like 'video' to the container + def has_gpus(self): + for d in self._devices: + if d.host_device == "/dev/dri": + return True + return False + + def render(self) -> list[str]: + return sorted([d.render() for d in self._devices]) diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/dns.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/dns.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/dns.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/dns.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/environment.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/environment.py new file mode 100644 index 0000000000..056763ea80 --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/environment.py @@ -0,0 +1,112 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render +try: + from .error import RenderError + from .formatter import escape_dollar + from .resources import Resources +except ImportError: + from error import RenderError + from formatter import escape_dollar + from resources import Resources + + +class Environment: + def __init__(self, render_instance: "Render", resources: Resources): + self._render_instance = render_instance + self._resources = resources + # Stores variables that user defined + self._user_vars: dict[str, Any] = {} + # Stores variables that are automatically added (based on values) + self._auto_variables: dict[str, Any] = {} + # Stores variables that are added by the application developer + self._app_dev_variables: dict[str, Any] = {} + + self._skip_generic_variables: bool = render_instance.values.get("skip_generic_variables", False) + + self._auto_add_variables_from_values() + + def _auto_add_variables_from_values(self): + if not self._skip_generic_variables: + self._add_generic_variables() + self._add_nvidia_variables() + + def _add_generic_variables(self): + self._auto_variables["TZ"] = self._render_instance.values.get("TZ", "Etc/UTC") + self._auto_variables["UMASK"] = self._render_instance.values.get("UMASK", "002") + self._auto_variables["UMASK_SET"] = self._render_instance.values.get("UMASK", "002") + + run_as = self._render_instance.values.get("run_as", {}) + user = run_as.get("user") + group = run_as.get("group") + if user: + self._auto_variables["PUID"] = user + self._auto_variables["UID"] = user + self._auto_variables["USER_ID"] = user + if group: + self._auto_variables["PGID"] = group + self._auto_variables["GID"] = group + self._auto_variables["GROUP_ID"] = group + + def _add_nvidia_variables(self): + if self._resources._nvidia_ids: + self._auto_variables["NVIDIA_DRIVER_CAPABILITIES"] = "all" + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = ",".join(sorted(self._resources._nvidia_ids)) + else: + self._auto_variables["NVIDIA_VISIBLE_DEVICES"] = "void" + + def _format_value(self, v: Any) -> str: + value = str(v) + + # str(bool) returns "True" or "False", + # but we want "true" or "false" + if isinstance(v, bool): + value = value.lower() + return value + + def add_env(self, name: str, value: Any): + if not name: + raise RenderError(f"Environment variable name cannot be empty. [{name}]") + if name in self._app_dev_variables.keys(): + raise RenderError( + f"Found duplicate environment variable [{name}] in application developer environment variables." + ) + self._app_dev_variables[name] = value + + def add_user_envs(self, user_env: list[dict]): + for item in user_env: + if not item.get("name"): + raise RenderError(f"Environment variable name cannot be empty. [{item}]") + if item["name"] in self._user_vars.keys(): + raise RenderError( + f"Found duplicate environment variable [{item['name']}] in user environment variables." + ) + self._user_vars[item["name"]] = item.get("value") + + def has_variables(self): + return len(self._auto_variables) > 0 or len(self._user_vars) > 0 or len(self._app_dev_variables) > 0 + + def render(self): + result: dict[str, str] = {} + + # Add envs from auto variables + result.update({k: self._format_value(v) for k, v in self._auto_variables.items()}) + + # Track defined keys for faster lookup + defined_keys = set(result.keys()) + + # Add envs from application developer (prohibit overwriting auto variables) + for k, v in self._app_dev_variables.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined automatically from the library.") + result[k] = self._format_value(v) + defined_keys.add(k) + + # Add envs from user (prohibit overwriting app developer envs and auto variables) + for k, v in self._user_vars.items(): + if k in defined_keys: + raise RenderError(f"Environment variable [{k}] is already defined from the application developer.") + result[k] = self._format_value(v) + + return {k: escape_dollar(v) for k, v in result.items()} diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/error.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/error.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/error.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/error.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/formatter.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/formatter.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/formatter.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/formatter.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/functions.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/functions.py new file mode 100644 index 0000000000..7d082d8c46 --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/functions.py @@ -0,0 +1,149 @@ +import re +import copy +import bcrypt +import secrets +from base64 import b64encode +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .volume_sources import HostPathSource, IxVolumeSource +except ImportError: + from error import RenderError + from volume_sources import HostPathSource, IxVolumeSource + + +class Functions: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + + def _bcrypt_hash(self, password): + hashed = bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") + return hashed + + def _htpasswd(self, username, password): + hashed = self._bcrypt_hash(password) + return username + ":" + hashed + + def _secure_string(self, length): + return secrets.token_urlsafe(length) + + def _basic_auth(self, username, password): + return b64encode(f"{username}:{password}".encode("utf-8")).decode("utf-8") + + def _basic_auth_header(self, username, password): + return f"Basic {self._basic_auth(username, password)}" + + def _fail(self, message): + raise RenderError(message) + + def _camel_case(self, string): + return string.title() + + def _auto_cast(self, value): + try: + return int(value) + except ValueError: + pass + + try: + return float(value) + except ValueError: + pass + + if value.lower() in ["true", "false"]: + return value.lower() == "true" + + return value + + def _match_regex(self, value, regex): + if not re.match(regex, value): + return False + return True + + def _must_match_regex(self, value, regex): + if not self._match_regex(value, regex): + raise RenderError(f"Expected [{value}] to match [{regex}]") + return value + + def _is_boolean(self, string): + return string.lower() in ["true", "false"] + + def _is_number(self, string): + try: + float(string) + return True + except ValueError: + return False + + def _copy_dict(self, dict): + return copy.deepcopy(dict) + + def _merge_dicts(self, *dicts): + merged_dict = {} + for dictionary in dicts: + merged_dict.update(dictionary) + return merged_dict + + def _disallow_chars(self, string: str, chars: list[str], key: str): + for char in chars: + if char in string: + raise RenderError(f"Disallowed character [{char}] in [{key}]") + return string + + def _or_default(self, value, default): + if not value: + return default + return value + + def _temp_config(self, name): + if not name: + raise RenderError("Expected [name] to be set when calling [temp_config].") + return {"type": "temporary", "volume_config": {"volume_name": name}} + + def _get_host_path(self, storage): + source_type = storage.get("type", "") + if not source_type: + raise RenderError("Expected [type] to be set for volume mounts.") + + match source_type: + case "host_path": + mount_config = storage.get("host_path_config") + if mount_config is None: + raise RenderError("Expected [host_path_config] to be set for [host_path] type.") + host_source = HostPathSource(self._render_instance, mount_config).get() + return host_source + case "ix_volume": + mount_config = storage.get("ix_volume_config") + if mount_config is None: + raise RenderError("Expected [ix_volume_config] to be set for [ix_volume] type.") + ix_source = IxVolumeSource(self._render_instance, mount_config).get() + return ix_source + case _: + raise RenderError(f"Storage type [{source_type}] does not support host path.") + + def func_map(self): + # TODO: Check what is no longer used and remove + return { + "auto_cast": self._auto_cast, + "basic_auth_header": self._basic_auth_header, + "basic_auth": self._basic_auth, + "bcrypt_hash": self._bcrypt_hash, + "camel_case": self._camel_case, + "copy_dict": self._copy_dict, + "fail": self._fail, + "htpasswd": self._htpasswd, + "is_boolean": self._is_boolean, + "is_number": self._is_number, + "match_regex": self._match_regex, + "merge_dicts": self._merge_dicts, + "must_match_regex": self._must_match_regex, + "secure_string": self._secure_string, + "disallow_chars": self._disallow_chars, + "get_host_path": self._get_host_path, + "or_default": self._or_default, + "temp_config": self._temp_config, + } diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/healthcheck.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/healthcheck.py new file mode 100644 index 0000000000..0805329284 --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/healthcheck.py @@ -0,0 +1,203 @@ +from typing import Any, TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .formatter import escape_dollar + from .validations import valid_http_path_or_raise +except ImportError: + from error import RenderError + from formatter import escape_dollar + from validations import valid_http_path_or_raise + + +class Healthcheck: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._test: str | list[str] = "" + self._interval_sec: int = 10 + self._timeout_sec: int = 5 + self._retries: int = 30 + self._start_period_sec: int = 10 + self._disabled: bool = False + self._use_built_in: bool = False + + def _get_test(self): + if isinstance(self._test, str): + return escape_dollar(self._test) + + return [escape_dollar(t) for t in self._test] + + def disable(self): + self._disabled = True + + def use_built_in(self): + self._use_built_in = True + + def set_custom_test(self, test: str | list[str]): + if self._disabled: + raise RenderError("Cannot set custom test when healthcheck is disabled") + self._test = test + + def set_test(self, variant: str, config: dict | None = None): + config = config or {} + self.set_custom_test(test_mapping(variant, config)) + + def set_interval(self, interval: int): + self._interval_sec = interval + + def set_timeout(self, timeout: int): + self._timeout_sec = timeout + + def set_retries(self, retries: int): + self._retries = retries + + def set_start_period(self, start_period: int): + self._start_period_sec = start_period + + def has_healthcheck(self): + return not self._use_built_in + + def render(self): + if self._use_built_in: + return RenderError("Should not be called when built in healthcheck is used") + + if self._disabled: + return {"disable": True} + + if not self._test: + raise RenderError("Healthcheck test is not set") + + return { + "test": self._get_test(), + "interval": f"{self._interval_sec}s", + "timeout": f"{self._timeout_sec}s", + "retries": self._retries, + "start_period": f"{self._start_period_sec}s", + } + + +def test_mapping(variant: str, config: dict | None = None) -> str: + config = config or {} + tests = { + "curl": curl_test, + "wget": wget_test, + "http": http_test, + "netcat": netcat_test, + "tcp": tcp_test, + "redis": redis_test, + "postgres": postgres_test, + "mariadb": mariadb_test, + } + + if variant not in tests: + raise RenderError(f"Test variant [{variant}] is not valid. Valid options are: [{', '.join(tests.keys())}]") + + return tests[variant](config) + + +def get_key(config: dict, key: str, default: Any, required: bool): + if not config.get(key): + if not required: + return default + raise RenderError(f"Expected [{key}] to be set") + return config[key] + + +def curl_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--insecure") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for curl test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "curl --silent --output /dev/null --show-error --fail" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def wget_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + scheme = get_key(config, "scheme", "http", False) + host = get_key(config, "host", "127.0.0.1", False) + headers = get_key(config, "headers", [], False) + + opts = [] + if scheme == "https": + opts.append("--no-check-certificate") + + for header in headers: + if not header[0] or not header[1]: + raise RenderError("Expected [header] to be a list of two items for wget test") + opts.append(f'--header "{header[0]}: {header[1]}"') + + cmd = "wget --spider --quiet" + if opts: + cmd += f" {' '.join(opts)}" + cmd += f" {scheme}://{host}:{port}{path}" + return cmd + + +def http_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + path = valid_http_path_or_raise(get_key(config, "path", "/", False)) + host = get_key(config, "host", "127.0.0.1", False) + + return f"""/bin/bash -c 'exec {{hc_fd}}<>/dev/tcp/{host}/{port} && echo -e "GET {path} HTTP/1.1\\r\\nHost: {host}\\r\\nConnection: close\\r\\n\\r\\n" >&${{hc_fd}} && cat <&${{hc_fd}} | grep "HTTP" | grep -q "200"'""" # noqa + + +def netcat_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"nc -z -w 1 {host} {port}" + + +def tcp_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", None, True) + host = get_key(config, "host", "127.0.0.1", False) + + return f"timeout 1 bash -c 'cat < /dev/null > /dev/tcp/{host}/{port}'" + + +def redis_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 6379, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"redis-cli -h {host} -p {port} -a $REDIS_PASSWORD ping | grep -q PONG" + + +def postgres_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 5432, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"pg_isready -h {host} -p {port} -U $POSTGRES_USER -d $POSTGRES_DB" + + +def mariadb_test(config: dict) -> str: + config = config or {} + port = get_key(config, "port", 3306, False) + host = get_key(config, "host", "127.0.0.1", False) + + return f"mariadb-admin --user=root --host={host} --port={port} --password=$MARIADB_ROOT_PASSWORD ping" diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/labels.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/labels.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/labels.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/labels.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/notes.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/notes.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/notes.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/notes.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/portal.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/portal.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/portal.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/portal.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/portals.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/portals.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/portals.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/portals.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/ports.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/ports.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/ports.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/ports.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/render.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/render.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/render.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/render.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/resources.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/resources.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/resources.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/resources.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/restart.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/restart.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/restart.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/restart.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/storage.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/storage.py new file mode 100644 index 0000000000..e697ba902a --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/storage.py @@ -0,0 +1,116 @@ +from typing import TYPE_CHECKING, TypedDict, Literal, NotRequired, Union + +if TYPE_CHECKING: + from render import Render + +try: + from .error import RenderError + from .validations import valid_fs_path_or_raise + from .volume_mount import VolumeMount +except ImportError: + from error import RenderError + from validations import valid_fs_path_or_raise + from volume_mount import VolumeMount + + +class IxStorageTmpfsConfig(TypedDict): + size: NotRequired[int] + mode: NotRequired[str] + + +class AclConfig(TypedDict, total=False): + path: str + + +class IxStorageHostPathConfig(TypedDict): + path: NotRequired[str] # Either this or acl.path must be set + acl_enable: NotRequired[bool] + acl: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageIxVolumeConfig(TypedDict): + dataset_name: str + acl_enable: NotRequired[bool] + acl_entries: NotRequired[AclConfig] + create_host_path: NotRequired[bool] + propagation: NotRequired[Literal["shared", "slave", "private", "rshared", "rslave", "rprivate"]] + auto_permissions: NotRequired[bool] # Only when acl_enable is false + + +class IxStorageVolumeConfig(TypedDict): + volume_name: NotRequired[str] + nocopy: NotRequired[bool] + auto_permissions: NotRequired[bool] + + +class IxStorageNfsConfig(TypedDict): + server: str + path: str + options: NotRequired[list[str]] + + +class IxStorageCifsConfig(TypedDict): + server: str + path: str + username: str + password: str + domain: NotRequired[str] + options: NotRequired[list[str]] + + +IxStorageVolumeLikeConfigs = Union[IxStorageVolumeConfig, IxStorageNfsConfig, IxStorageCifsConfig, IxStorageTmpfsConfig] +IxStorageBindLikeConfigs = Union[IxStorageHostPathConfig, IxStorageIxVolumeConfig] +IxStorageLikeConfigs = Union[IxStorageBindLikeConfigs, IxStorageVolumeLikeConfigs] + + +class IxStorage(TypedDict): + type: Literal["ix_volume", "host_path", "tmpfs", "volume", "anonymous", "temporary"] + read_only: NotRequired[bool] + + ix_volume_config: NotRequired[IxStorageIxVolumeConfig] + host_path_config: NotRequired[IxStorageHostPathConfig] + tmpfs_config: NotRequired[IxStorageTmpfsConfig] + volume_config: NotRequired[IxStorageVolumeConfig] + nfs_config: NotRequired[IxStorageNfsConfig] + cifs_config: NotRequired[IxStorageCifsConfig] + + +class Storage: + def __init__(self, render_instance: "Render"): + self._render_instance = render_instance + self._volume_mounts: set[VolumeMount] = set() + + def add(self, mount_path: str, config: "IxStorage"): + mount_path = valid_fs_path_or_raise(mount_path) + if mount_path in [m.mount_path for m in self._volume_mounts]: + raise RenderError(f"Mount path [{mount_path}] already used for another volume mount") + + volume_mount = VolumeMount(self._render_instance, mount_path, config) + self._volume_mounts.add(volume_mount) + + def _add_docker_socket(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/var/run/docker.sock", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def _add_tun_device(self, read_only: bool = True, mount_path: str = ""): + mount_path = valid_fs_path_or_raise(mount_path) + cfg: "IxStorage" = { + "type": "host_path", + "read_only": read_only, + "host_path_config": {"path": "/dev/net/tun", "create_host_path": False}, + } + self.add(mount_path, cfg) + + def has_mounts(self) -> bool: + return bool(self._volume_mounts) + + def render(self): + return [vm.render() for vm in sorted(self._volume_mounts, key=lambda vm: vm.mount_path)] diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/sysctls.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/sysctls.py new file mode 100644 index 0000000000..e6b8469f3b --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/sysctls.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from render import Render + from container import Container + +try: + from .error import RenderError + from .validations import valid_sysctl_or_raise +except ImportError: + from error import RenderError + from validations import valid_sysctl_or_raise + + +class Sysctls: + def __init__(self, render_instance: "Render", container_instance: "Container"): + self._render_instance = render_instance + self._container_instance = container_instance + self._sysctls: dict = {} + + def add(self, key: str, value): + key = key.strip() + if not key: + raise RenderError("Sysctls key cannot be empty") + if value is None: + raise RenderError(f"Sysctl [{key}] requires a value") + if key in self._sysctls: + raise RenderError(f"Sysctl [{key}] already added") + self._sysctls[key] = str(value) + + def has_sysctls(self): + return bool(self._sysctls) + + def render(self): + if not self.has_sysctls(): + return {} + host_net = self._container_instance._network_mode == "host" + return {valid_sysctl_or_raise(k, host_net): v for k, v in self._sysctls.items()} diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/__init__.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/__init__.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/__init__.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/__init__.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_build_image.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_build_image.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_build_image.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_build_image.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_configs.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_configs.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_configs.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_configs.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_container.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_container.py new file mode 100644 index 0000000000..747ad39357 --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_container.py @@ -0,0 +1,360 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_empty_container_name(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container(" ", "test_image") + + +def test_resolve_image(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["image"] == "nginx:latest" + + +def test_missing_repo(mock_values): + mock_values["images"]["test_image"]["repository"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_missing_tag(mock_values): + mock_values["images"]["test_image"]["tag"] = "" + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "test_image") + + +def test_non_existing_image(mock_values): + render = Render(mock_values) + with pytest.raises(Exception): + render.add_container("test_container", "non_existing_image") + + +def test_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_pull_policy("always") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["pull_policy"] == "always" + + +def test_invalid_pull_policy(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.set_pull_policy("invalid_policy") + + +def test_clear_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.add_caps(["NET_ADMIN"]) + c1.clear_caps() + c1.healthcheck.disable() + output = render.render() + assert "cap_drop" not in output["services"]["test_container"] + assert "cap_add" not in output["services"]["test_container"] + + +def test_privileged(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_privileged(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["privileged"] is True + + +def test_tty(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_tty(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["tty"] is True + + +def test_init(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_init(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["init"] is True + + +def test_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_read_only(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["read_only"] is True + + +def test_stdin(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_stdin(True) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stdin_open"] is True + + +def test_hostname(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_hostname("test_hostname") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["hostname"] == "test_hostname" + + +def test_grace_period(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_grace_period(10) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["stop_grace_period"] == "10s" + + +def test_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_user(1000, 1000) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["user"] == "1000:1000" + + +def test_invalid_user(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_user(-100, 1000) + + +def test_add_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + c1.add_group("video") + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 1000, "video"] + + +def test_add_duplicate_group(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_group(1000) + with pytest.raises(Exception): + c1.add_group(1000) + + +def test_add_group_as_string(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_group("1000") + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_docker_socket() + output = render.render() + assert output["services"]["test_container"]["group_add"] == [568, 999] + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_tun_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_tun_device() + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/dev/net/tun", + "target": "/dev/net/tun", + "read_only": True, + "bind": { + "propagation": "rprivate", + "create_host_path": False, + }, + } + ] + + +def test_snd_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_snd_device() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/snd:/dev/snd"] + assert output["services"]["test_container"]["group_add"] == [29, 568] + + +def test_shm_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_shm_size_mb(10) + output = render.render() + assert output["services"]["test_container"]["shm_size"] == "10M" + + +def test_valid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_caps(["ALL", "NET_ADMIN"]) + output = render.render() + assert output["services"]["test_container"]["cap_add"] == ["ALL", "NET_ADMIN"] + assert output["services"]["test_container"]["cap_drop"] == ["ALL"] + + +def test_add_duplicate_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["ALL", "NET_ADMIN", "NET_ADMIN"]) + + +def test_invalid_caps(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_caps(["invalid_cap"]) + + +def test_remove_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.remove_security_opt("no-new-privileges") + output = render.render() + assert "security_opt" not in output["services"]["test_container"] + + +def test_add_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_security_opt("seccomp=unconfined") + output = render.render() + assert output["services"]["test_container"]["security_opt"] == [ + "no-new-privileges", + "seccomp=unconfined", + ] + + +def test_add_duplicate_security_opt(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_security_opt("no-new-privileges") + + +def test_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_auto_network_mode_with_host_network(mock_values): + mock_values["network"] = {"host_network": True} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "host" + + +def test_network_mode_with_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("service:test_container") + output = render.render() + assert output["services"]["test_container"]["network_mode"] == "service:test_container" + + +def test_network_mode_with_container_missing(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("service:missing_container") + + +def test_invalid_network_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.set_network_mode("invalid_mode") + + +def test_entrypoint(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_entrypoint(["/bin/bash", "-c", "echo hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["entrypoint"] == ["/bin/bash", "-c", "echo hello $$MY_ENV"] + + +def test_command(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.set_command(["echo", "hello $MY_ENV"]) + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["command"] == ["echo", "hello $$MY_ENV"] diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_depends.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_depends.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_depends.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_depends.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_deps.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_deps.py new file mode 100644 index 0000000000..a1b7f03a60 --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_deps.py @@ -0,0 +1,477 @@ +import json +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_postgres_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "16"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + p = render.deps.postgres( + "pg_container", + "pg_image", + { + "user": "test_user", + "password": "test_@password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + p.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert ( + p.get_url("postgres") == "postgres://test_user:test_%40password@pg_container:5432/test_database?sslmode=disable" + ) + assert "devices" not in output["services"]["pg_container"] + assert "reservations" not in output["services"]["pg_container"]["deploy"]["resources"] + assert output["services"]["pg_container"]["image"] == "postgres:16" + assert output["services"]["pg_container"]["user"] == "999:999" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["pg_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["pg_container"]["healthcheck"] == { + "test": "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["pg_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/postgresql/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["pg_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_@password", + "POSTGRES_DB": "test_database", + "POSTGRES_PORT": "5432", + } + assert output["services"]["pg_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"}, + "pg_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert output["services"]["perms_container"]["restart"] == "on-failure:1" + + +def test_add_redis_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test_password", "volume": {}}, # type: ignore + ) + + +def test_add_redis_with_password_with_spaces(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.redis( + "test_container", + "test_image", + {"password": "test password", "volume": {}}, # type: ignore + ) + + +def test_add_redis(mock_values): + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + r = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test&password@", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + c1.environment.add_env("REDIS_URL", r.get_url("redis")) + if perms_container.has_actions(): + perms_container.activate() + r.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["redis_container"] + assert "reservations" not in output["services"]["redis_container"]["deploy"]["resources"] + assert ( + output["services"]["test_container"]["environment"]["REDIS_URL"] + == "redis://default:test%26password%40@redis_container:6379" + ) + assert output["services"]["redis_container"]["image"] == "redis:latest" + assert output["services"]["redis_container"]["user"] == "1001:0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["redis_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["redis_container"]["healthcheck"] == { + "test": "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["redis_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/bitnami/redis/data", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["redis_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "ALLOW_EMPTY_PASSWORD": "no", + "REDIS_PASSWORD": "test&password@", + "REDIS_PORT_NUMBER": "6379", + } + assert output["services"]["redis_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_mariadb_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.mariadb( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_mariadb(mock_values): + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("perms_container") + m = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + m.container.depends.add_dependency("perms_container", "service_completed_successfully") + output = render.render() + assert "devices" not in output["services"]["mariadb_container"] + assert "reservations" not in output["services"]["mariadb_container"]["deploy"]["resources"] + assert output["services"]["mariadb_container"]["image"] == "mariadb:latest" + assert output["services"]["mariadb_container"]["user"] == "999:999" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["cpus"] == "2.0" + assert output["services"]["mariadb_container"]["deploy"]["resources"]["limits"]["memory"] == "4096M" + assert output["services"]["mariadb_container"]["healthcheck"] == { + "test": "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + assert output["services"]["mariadb_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/var/lib/mysql", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["services"]["mariadb_container"]["environment"] == { + "TZ": "Etc/UTC", + "UMASK": "002", + "UMASK_SET": "002", + "NVIDIA_VISIBLE_DEVICES": "void", + "MARIADB_USER": "test_user", + "MARIADB_PASSWORD": "test_password", + "MARIADB_ROOT_PASSWORD": "test_password", + "MARIADB_DATABASE": "test_database", + "MARIADB_AUTO_UPGRADE": "true", + } + assert output["services"]["mariadb_container"]["depends_on"] == { + "perms_container": {"condition": "service_completed_successfully"} + } + + +def test_add_perms_container(mock_values): + mock_values["ix_volumes"] = { + "test_dataset1": "/mnt/test/1", + "test_dataset2": "/mnt/test/2", + "test_dataset3": "/mnt/test/3", + } + mock_values["images"]["postgres_image"] = {"repository": "postgres", "tag": "17"} + mock_values["images"]["redis_image"] = {"repository": "redis", "tag": "latest"} + mock_values["images"]["mariadb_image"] = {"repository": "mariadb", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + # fmt: off + volume_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + volume_no_perms = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + host_path_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "auto_permissions": True}} + host_path_no_perms = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + host_path_acl_perms = {"type": "host_path", "host_path_config": {"acl": {"path": "/mnt/test"}, "acl_enable": True, "auto_permissions": True}} # noqa + ix_volume_no_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset1"}} + ix_volume_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset2", "auto_permissions": True}} # noqa + ix_volume_acl_perms = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset3", "acl_enable": True, "auto_permissions": True}} # noqa + temp_volume = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + # fmt: on + + c1.add_storage("/some/path1", volume_perms) + c1.add_storage("/some/path2", volume_no_perms) + c1.add_storage("/some/path3", host_path_perms) + c1.add_storage("/some/path4", host_path_no_perms) + c1.add_storage("/some/path5", host_path_acl_perms) + c1.add_storage("/some/path6", ix_volume_no_perms) + c1.add_storage("/some/path7", ix_volume_perms) + c1.add_storage("/some/path8", ix_volume_acl_perms) + c1.add_storage("/some/path9", temp_volume) + + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data2", volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data3", host_path_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data4", host_path_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data5", host_path_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data6", ix_volume_no_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data7", ix_volume_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data8", ix_volume_acl_perms, {"uid": 1000, "gid": 1000, "mode": "check"}) + perms_container.add_or_skip_action("data9", temp_volume, {"uid": 1000, "gid": 1000, "mode": "check"}) + + postgres = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + redis = render.deps.redis( + "redis_container", + "redis_image", + { + "password": "test_password", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + mariadb = render.deps.mariadb( + "mariadb_container", + "mariadb_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + postgres.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + redis.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + mariadb.container.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert output["services"]["test_perms_container"]["network_mode"] == "none" + assert output["services"]["test_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + assert output["configs"]["permissions_run_script"]["content"] != "" + # fmt: off + content = [ + {"mount_path": "/mnt/permission/data", "is_temporary": False, "identifier": "data", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data3", "is_temporary": False, "identifier": "data3", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data6", "is_temporary": False, "identifier": "data6", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data7", "is_temporary": False, "identifier": "data7", "recursive": False, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/data9", "is_temporary": True, "identifier": "data9", "recursive": True, "mode": "check", "uid": 1000, "gid": 1000, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/postgres_container_postgres_data", "is_temporary": False, "identifier": "postgres_container_postgres_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/redis_container_redis_data", "is_temporary": False, "identifier": "redis_container_redis_data", "recursive": False, "mode": "check", "uid": 1001, "gid": 0, "chmod": None}, # noqa + {"mount_path": "/mnt/permission/mariadb_container_mariadb_data", "is_temporary": False, "identifier": "mariadb_container_mariadb_data", "recursive": False, "mode": "check", "uid": 999, "gid": 999, "chmod": None}, # noqa + ] + # fmt: on + assert output["configs"]["permissions_actions_data"]["content"] == json.dumps(content) + + +def test_add_duplicate_perms_action(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + with pytest.raises(Exception): + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + + +def test_add_perm_action_without_auto_perms_enabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": False}} + c1.add_storage("/some/path", vol_config) + perms_container = render.deps.perms("test_perms_container") + perms_container.add_or_skip_action("data", vol_config, {"uid": 1000, "gid": 1000, "mode": "check"}) + if perms_container.has_actions(): + perms_container.activate() + c1.depends.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert "configs" not in output + assert "ix-test_perms_container" not in output["services"] + assert "depends_on" not in output["services"]["test_container"] + + +def test_add_unsupported_postgres_version(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "99"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_add_postgres_with_invalid_tag(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": "latest"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + render.deps.postgres( + "test_container", + "test_image", + {"user": "test_user", "password": "test_password", "database": "test_database"}, # type: ignore + ) + + +def test_no_upgrade_container_with_non_postgres_image(mock_values): + mock_values["images"]["postgres_image"] = {"repository": "tensorchord/pgvecto-rs", "tag": "pg15-v0.2.0"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "postgres_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + assert len(output["services"]) == 3 # c1, pg, perms + assert output["services"]["postgres_container"]["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"} + } + + +def test_postgres_with_upgrade_container(mock_values): + mock_values["images"]["pg_image"] = {"repository": "postgres", "tag": 16.6} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + perms_container = render.deps.perms("test_perms_container") + pg = render.deps.postgres( + "postgres_container", + "pg_image", + { + "user": "test_user", + "password": "test_password", + "database": "test_database", + "volume": {"type": "volume", "volume_config": {"volume_name": "test_volume", "auto_permissions": True}}, + }, + perms_container, + ) + if perms_container.has_actions(): + perms_container.activate() + pg.add_dependency("test_perms_container", "service_completed_successfully") + output = render.render() + pg = output["services"]["postgres_container"] + pgup = output["services"]["postgres_container_upgrade"] + assert pg["volumes"] == pgup["volumes"] + assert pg["user"] == pgup["user"] + assert pgup["environment"]["TARGET_VERSION"] == "16" + assert pgup["environment"]["DATA_DIR"] == "/var/lib/postgresql/data" + pgup_env = pgup["environment"] + pgup_env.pop("TARGET_VERSION") + pgup_env.pop("DATA_DIR") + assert pg["environment"] == pgup_env + assert pg["depends_on"] == { + "test_perms_container": {"condition": "service_completed_successfully"}, + "postgres_container_upgrade": {"condition": "service_completed_successfully"}, + } + assert pgup["depends_on"] == {"test_perms_container": {"condition": "service_completed_successfully"}} + assert pgup["restart"] == "on-failure:1" + assert pgup["healthcheck"] == {"disable": True} + assert pgup["build"]["dockerfile_inline"] != "" + assert pgup["configs"][0]["source"] == "pg_container_upgrade.sh" + assert pgup["entrypoint"] == ["/bin/bash", "-c", "/upgrade.sh"] diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_device.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_device.py new file mode 100644 index 0000000000..c44437367d --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_device.py @@ -0,0 +1,131 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + c1.devices.add_device("/h/dev/sdb", "/c/dev/sdb", "rwm") + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/h/dev/sda:/c/dev/sda", "/h/dev/sdb:/c/dev/sdb:rwm"] + + +def test_devices_without_host(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("", "/c/dev/sda") + + +def test_devices_without_container(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "") + + +def test_add_duplicate_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda") + + +def test_add_device_with_invalid_container_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "c/dev/sda") + + +def test_add_device_with_invalid_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("h/dev/sda", "/c/dev/sda") + + +def test_add_disallowed_device(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/dri", "/c/dev/sda") + + +def test_add_device_with_invalid_cgroup_perm(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/h/dev/sda", "/c/dev/sda", "invalid") + + +def test_automatically_add_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_automatically_add_gpu_devices_and_kfd(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True, "kfd_device_exists": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/dri:/dev/dri", "/dev/kfd:/dev/kfd"] + assert output["services"]["test_container"]["group_add"] == [44, 107, 568] + + +def test_remove_gpu_devices(mock_values): + mock_values["resources"] = {"gpus": {"use_all_gpus": True}} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.remove_devices() + output = render.render() + assert "devices" not in output["services"]["test_container"] + assert output["services"]["test_container"]["group_add"] == [568] + + +def test_add_usb_bus(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.devices.add_usb_bus() + output = render.render() + assert output["services"]["test_container"]["devices"] == ["/dev/bus/usb:/dev/bus/usb"] + + +def test_add_usb_bus_disallowed(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.devices.add_device("/dev/bus/usb", "/dev/bus/usb") diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_dns.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_dns.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_dns.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_dns.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_environment.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_environment.py new file mode 100644 index 0000000000..d657646582 --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_environment.py @@ -0,0 +1,196 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_auto_add_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + mock_values["run_as"] = {"user": "1000", "group": "1000"} + mock_values["resources"] = { + "gpus": { + "nvidia_gpu_selection": { + "pci_slot_0": {"uuid": "uuid_0", "use_gpu": True}, + "pci_slot_1": {"uuid": "uuid_1", "use_gpu": True}, + }, + } + } + + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert len(envs) == 11 + assert envs["TZ"] == "Etc/UTC" + assert envs["PUID"] == "1000" + assert envs["UID"] == "1000" + assert envs["USER_ID"] == "1000" + assert envs["PGID"] == "1000" + assert envs["GID"] == "1000" + assert envs["GROUP_ID"] == "1000" + assert envs["UMASK"] == "002" + assert envs["UMASK_SET"] == "002" + assert envs["NVIDIA_DRIVER_CAPABILITIES"] == "all" + assert envs["NVIDIA_VISIBLE_DEVICES"] == "uuid_0,uuid_1" + + +def test_skip_generic_variables(mock_values): + mock_values["skip_generic_variables"] = True + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + envs = output["services"]["test_container"]["environment"] + + assert len(envs) == 1 + assert envs["NVIDIA_VISIBLE_DEVICES"] == "void" + + +def test_add_from_all_sources(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_value") + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_value" + assert envs["USER_ENV"] == "test_value2" + assert envs["TZ"] == "Etc/UTC" + + +def test_user_add_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV2", "value": "test_value2"}, + ] + ) + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["MY_ENV"] == "test_value" + assert envs["MY_ENV2"] == "test_value2" + + +def test_user_add_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "MY_ENV", "value": "test_value"}, + {"name": "MY_ENV", "value": "test_value2"}, + ] + ) + + +def test_user_env_without_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_user_envs( + [ + {"name": "", "value": "test_value"}, + ] + ) + + +def test_user_env_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "TZ", "value": "test_value"}, + ] + ) + with pytest.raises(Exception): + render.render() + + +def test_user_env_try_to_overwrite_app_dev_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_user_envs( + [ + {"name": "PORT", "value": "test_value"}, + ] + ) + c1.environment.add_env("PORT", "test_value2") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_vars_try_to_overwrite_auto_vars(mock_values): + mock_values["TZ"] = "Etc/UTC" + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("TZ", "test_value") + with pytest.raises(Exception): + render.render() + + +def test_app_dev_no_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.environment.add_env("", "test_value") + + +def test_app_dev_duplicate_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("PORT", "test_value") + with pytest.raises(Exception): + c1.environment.add_env("PORT", "test_value2") + + +def test_format_vars(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.environment.add_env("APP_ENV", "test_$value") + c1.environment.add_env("APP_ENV_BOOL", True) + c1.environment.add_env("APP_ENV_INT", 10) + c1.environment.add_env("APP_ENV_FLOAT", 10.5) + c1.environment.add_user_envs( + [ + {"name": "USER_ENV", "value": "test_$value2"}, + ] + ) + + output = render.render() + envs = output["services"]["test_container"]["environment"] + assert envs["APP_ENV"] == "test_$$value" + assert envs["USER_ENV"] == "test_$$value2" + assert envs["APP_ENV_BOOL"] == "true" + assert envs["APP_ENV_INT"] == "10" + assert envs["APP_ENV_FLOAT"] == "10.5" diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_formatter.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_formatter.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_formatter.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_formatter.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_functions.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_functions.py new file mode 100644 index 0000000000..0ea3b57d18 --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_functions.py @@ -0,0 +1,88 @@ +import re +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_funcs(mock_values): + mock_values["ix_volumes"] = {"test": "/mnt/test123"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + + tests = [ + {"func": "auto_cast", "values": ["1"], "expected": 1}, + {"func": "basic_auth_header", "values": ["my_user", "my_pass"], "expected": "Basic bXlfdXNlcjpteV9wYXNz"}, + {"func": "basic_auth", "values": ["my_user", "my_pass"], "expected": "bXlfdXNlcjpteV9wYXNz"}, + { + "func": "bcrypt_hash", + "values": ["my_pass"], + "expect_regex": r"^\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "camel_case", "values": ["my_user"], "expected": "My_User"}, + {"func": "copy_dict", "values": [{"a": 1}], "expected": {"a": 1}}, + {"func": "fail", "values": ["my_message"], "expect_raise": True}, + { + "func": "htpasswd", + "values": ["my_user", "my_pass"], + "expect_regex": r"^my_user:\$2b\$12\$[a-zA-Z0-9-_\.\/]+$", + }, + {"func": "is_boolean", "values": ["true"], "expected": True}, + {"func": "is_boolean", "values": ["false"], "expected": True}, + {"func": "is_number", "values": ["1"], "expected": True}, + {"func": "is_number", "values": ["1.1"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[a-zA-Z0-9]+$"], "expected": True}, + {"func": "match_regex", "values": ["value", "^[0-9]+$"], "expected": False}, + {"func": "merge_dicts", "values": [{"a": 1}, {"b": 2}], "expected": {"a": 1, "b": 2}}, + {"func": "must_match_regex", "values": ["my_user", "^[0-9]$"], "expect_raise": True}, + {"func": "must_match_regex", "values": ["1", "^[0-9]$"], "expected": "1"}, + {"func": "secure_string", "values": [10], "expect_regex": r"^[a-zA-Z0-9-_]+$"}, + {"func": "disallow_chars", "values": ["my_user", ["$", "@"], "my_key"], "expected": "my_user"}, + {"func": "disallow_chars", "values": ["my_user$", ["$", "@"], "my_key"], "expect_raise": True}, + { + "func": "get_host_path", + "values": [{"type": "host_path", "host_path_config": {"path": "/mnt/test"}}], + "expected": "/mnt/test", + }, + { + "func": "get_host_path", + "values": [{"type": "ix_volume", "ix_volume_config": {"dataset_name": "test"}}], + "expected": "/mnt/test123", + }, + {"func": "or_default", "values": [None, 1], "expected": 1}, + {"func": "or_default", "values": [1, None], "expected": 1}, + {"func": "or_default", "values": [False, 1], "expected": 1}, + {"func": "or_default", "values": [True, 1], "expected": True}, + {"func": "temp_config", "values": [""], "expect_raise": True}, + { + "func": "temp_config", + "values": ["test"], + "expected": {"type": "temporary", "volume_config": {"volume_name": "test"}}, + }, + ] + + for test in tests: + print(test["func"], test) + func = render.funcs[test["func"]] + if test.get("expect_raise", False): + with pytest.raises(Exception): + func(*test["values"]) + elif test.get("expect_regex"): + r = func(*test["values"]) + assert re.match(test["expect_regex"], r) is not None + else: + r = func(*test["values"]) + assert r == test["expected"] diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_healthcheck.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_healthcheck.py new file mode 100644 index 0000000000..8fa044290f --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_healthcheck.py @@ -0,0 +1,195 @@ +import pytest + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_disable_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == {"disable": True} + + +def test_use_built_in_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.use_built_in() + output = render.render() + assert "healthcheck" not in output["services"]["test_container"] + + +def test_set_custom_test(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test("echo $1") + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": "echo $$1", + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_custom_test_array(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "10s", + "timeout": "5s", + "retries": 30, + "start_period": "10s", + } + + +def test_set_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_custom_test(["CMD", "echo", "$1"]) + c1.healthcheck.set_interval(9) + c1.healthcheck.set_timeout(8) + c1.healthcheck.set_retries(7) + c1.healthcheck.set_start_period(6) + output = render.render() + assert output["services"]["test_container"]["healthcheck"] == { + "test": ["CMD", "echo", "$$1"], + "interval": "9s", + "timeout": "8s", + "retries": 7, + "start_period": "6s", + } + + +def test_adding_test_when_disabled(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.healthcheck.set_custom_test("echo $1") + + +def test_not_adding_test(mock_values): + render = Render(mock_values) + render.add_container("test_container", "test_image") + with pytest.raises(Exception): + render.render() + + +def test_invalid_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + with pytest.raises(Exception): + c1.healthcheck.set_test("http", {"port": 8080, "path": "invalid"}) + + +def test_http_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("http", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == """/bin/bash -c 'exec {hc_fd}<>/dev/tcp/127.0.0.1/8080 && echo -e "GET / HTTP/1.1\\r\\nHost: 127.0.0.1\\r\\nConnection: close\\r\\n\\r\\n" >&$${hc_fd} && cat <&$${hc_fd} | grep "HTTP" | grep -q "200"'""" # noqa + ) + + +def test_curl_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "curl --silent --output /dev/null --show-error --fail http://127.0.0.1:8080/health" + ) + + +def test_curl_healthcheck_with_headers(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("curl", {"port": 8080, "path": "/health", "headers": [("X-Test", "$test")]}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == 'curl --silent --output /dev/null --show-error --fail --header "X-Test: $$test" http://127.0.0.1:8080/health' + ) + + +def test_wget_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("wget", {"port": 8080, "path": "/health"}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "wget --spider --quiet http://127.0.0.1:8080/health" + ) + + +def test_netcat_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("netcat", {"port": 8080}) + output = render.render() + assert output["services"]["test_container"]["healthcheck"]["test"] == "nc -z -w 1 127.0.0.1 8080" + + +def test_tcp_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("tcp", {"port": 8080}) + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "timeout 1 bash -c 'cat < /dev/null > /dev/tcp/127.0.0.1/8080'" + ) + + +def test_redis_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("redis") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "redis-cli -h 127.0.0.1 -p 6379 -a $$REDIS_PASSWORD ping | grep -q PONG" + ) + + +def test_postgres_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("postgres") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "pg_isready -h 127.0.0.1 -p 5432 -U $$POSTGRES_USER -d $$POSTGRES_DB" + ) + + +def test_mariadb_healthcheck(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.set_test("mariadb") + output = render.render() + assert ( + output["services"]["test_container"]["healthcheck"]["test"] + == "mariadb-admin --user=root --host=127.0.0.1 --port=3306 --password=$$MARIADB_ROOT_PASSWORD ping" + ) diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_labels.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_labels.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_labels.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_labels.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_notes.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_notes.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_notes.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_notes.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_portal.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_portal.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_portal.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_portal.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_ports.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_ports.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_ports.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_ports.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_render.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_render.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_render.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_render.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_resources.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_resources.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_resources.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_resources.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_restart.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_restart.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/tests/test_restart.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_restart.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_sysctls.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_sysctls.py new file mode 100644 index 0000000000..c9414044ea --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_sysctls.py @@ -0,0 +1,62 @@ +import pytest + + +from render import Render + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + c1.sysctls.add("fs.mqueue.msg_max", 100) + output = render.render() + assert output["services"]["test_container"]["sysctls"] == {"net.ipv4.ip_forward": "1", "fs.mqueue.msg_max": "100"} + + +def test_add_net_sysctl_with_host_network(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.set_network_mode("host") + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + render.render() + + +def test_add_duplicate_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("net.ipv4.ip_forward", 1) + with pytest.raises(Exception): + c1.sysctls.add("net.ipv4.ip_forward", 0) + + +def test_add_empty_sysctl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.sysctls.add("", 1) + + +def test_add_sysctl_with_invalid_key(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.sysctls.add("invalid.sysctl", 1) + with pytest.raises(Exception): + render.render() diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_volumes.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_volumes.py new file mode 100644 index 0000000000..aef0d39481 --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/tests/test_volumes.py @@ -0,0 +1,666 @@ +import pytest + + +from render import Render +from formatter import get_hashed_name_for_volume + + +@pytest.fixture +def mock_values(): + return { + "images": { + "test_image": { + "repository": "nginx", + "tag": "latest", + } + }, + } + + +def test_add_volume_invalid_type(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "invalid_type"}) + + +def test_add_volume_empty_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + with pytest.raises(Exception): + c1.add_storage("", {"type": "tmpfs"}) + + +def test_add_volume_duplicate_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.add_storage("/some/path", {"type": "tmpfs"}) + with pytest.raises(Exception): + c1.add_storage("/some/path", {"type": "tmpfs"}) + + +def test_add_volume_host_path_invalid_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "propagation": "invalid_propagation"}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_host_path_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path"} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_with_acl_no_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"acl_enable": True, "acl": {"path": ""}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", host_path_config) + + +def test_add_host_path_volume_mount(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_acl(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = { + "type": "host_path", + "host_path_config": {"path": "/mnt/test", "acl_enable": True, "acl": {"path": "/mnt/test/acl"}}, + } + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test/acl", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_propagation(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "propagation": "slave"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "slave"}, + } + ] + + +def test_add_host_path_volume_mount_with_create_host_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "host_path_config": {"path": "/mnt/test", "create_host_path": True}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rprivate"}, + } + ] + + +def test_add_host_path_volume_mount_with_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + host_path_config = {"type": "host_path", "read_only": True, "host_path_config": {"path": "/mnt/test"}} + c1.add_storage("/some/path", host_path_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_invalid_dataset_name(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "invalid_dataset"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_no_ix_volume_config(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume"} + with pytest.raises(Exception): + c1.add_storage("/some/path", ix_volume_config) + + +def test_add_ix_volume_volume_mount(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = {"type": "ix_volume", "ix_volume_config": {"dataset_name": "test_dataset"}} + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_ix_volume_volume_mount_with_options(mock_values): + mock_values["ix_volumes"] = {"test_dataset": "/mnt/test"} + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + ix_volume_config = { + "type": "ix_volume", + "ix_volume_config": {"dataset_name": "test_dataset", "propagation": "rslave", "create_host_path": True}, + } + c1.add_storage("/some/path", ix_volume_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/mnt/test", + "target": "/some/path", + "read_only": False, + "bind": {"create_host_path": True, "propagation": "rslave"}, + } + ] + + +def test_cifs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"path": "/path", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "username": "user", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_username(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "password": "password"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_missing_password(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs", "cifs_config": {"server": "server", "path": "/path", "username": "user"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_without_cifs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = {"type": "cifs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["verbose=true", "verbose=true"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["user=username"], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": {"verbose": True}, + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_cifs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_config = { + "type": "cifs", + "cifs_config": { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": [{"verbose": True}], + }, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", cifs_config) + + +def test_add_cifs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = {"server": "server", "path": "/path", "username": "user", "password": "pas$word"} + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": {"type": "cifs", "device": "//server/path", "o": "noperm,password=pas$$word,user=user"} + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_cifs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + cifs_inner_config = { + "server": "server", + "path": "/path", + "username": "user", + "password": "pas$word", + "options": ["vers=3.0", "verbose=true"], + } + cifs_config = {"type": "cifs", "cifs_config": cifs_inner_config} + c1.add_storage("/some/path", cifs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("cifs", cifs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "cifs", + "device": "//server/path", + "o": "noperm,password=pas$$word,user=user,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_missing_server(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"path": "/path"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_missing_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_without_nfs_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs"} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_duplicate_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = { + "type": "nfs", + "nfs_config": {"server": "server", "path": "/path", "options": ["verbose=true", "verbose=true"]}, + } + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_disallowed_option(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": ["addr=server"]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": {"verbose": True}}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_nfs_volume_invalid_options2(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_config = {"type": "nfs", "nfs_config": {"server": "server", "path": "/path", "options": [{"verbose": True}]}} + with pytest.raises(Exception): + c1.add_storage("/some/path", nfs_config) + + +def test_add_nfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path"} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == {vol_name: {"driver_opts": {"type": "nfs", "device": ":/path", "o": "addr=server"}}} + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_nfs_volume_with_options(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + nfs_inner_config = {"server": "server", "path": "/path", "options": ["vers=3.0", "verbose=true"]} + nfs_config = {"type": "nfs", "nfs_config": nfs_inner_config} + c1.add_storage("/some/path", nfs_config) + output = render.render() + vol_name = get_hashed_name_for_volume("nfs", nfs_inner_config) + assert output["volumes"] == { + vol_name: { + "driver_opts": { + "type": "nfs", + "device": ":/path", + "o": "addr=server,verbose=true,vers=3.0", + } + } + } + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "source": vol_name, "target": "/some/path", "read_only": False, "volume": {"nocopy": False}} + ] + + +def test_tmpfs_invalid_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": "2"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_zero_size(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"size": 0}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_invalid_mode(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs", "tmpfs_config": {"mode": "invalid"}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_tmpfs_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "tmpfs"} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "tmpfs", + "target": "/some/path", + "read_only": False, + } + ] + + +def test_temporary_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "temporary", "volume_config": {"volume_name": "test_temp_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "source": "test_temp_volume", + "type": "volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + + +def test_docker_volume_missing_config(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume_missing_volume_name(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": ""}} + with pytest.raises(Exception): + c1.add_storage("/some/path", vol_config) + + +def test_docker_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "volume", "volume_config": {"volume_name": "test_volume"}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "volume", + "source": "test_volume", + "target": "/some/path", + "read_only": False, + "volume": {"nocopy": False}, + } + ] + assert output["volumes"] == {"test_volume": {}} + + +def test_anonymous_volume(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + vol_config = {"type": "anonymous", "volume_config": {"nocopy": True}} + c1.add_storage("/some/path", vol_config) + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + {"type": "volume", "target": "/some/path", "read_only": False, "volume": {"nocopy": True}} + ] + assert "volumes" not in output + + +def test_add_docker_socket(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_not_read_only(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(read_only=False, mount_path="/var/run/docker.sock") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/var/run/docker.sock", + "read_only": False, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] + + +def test_add_docker_socket_mount_path(mock_values): + render = Render(mock_values) + c1 = render.add_container("test_container", "test_image") + c1.healthcheck.disable() + c1.storage._add_docker_socket(mount_path="/some/path") + output = render.render() + assert output["services"]["test_container"]["volumes"] == [ + { + "type": "bind", + "source": "/var/run/docker.sock", + "target": "/some/path", + "read_only": True, + "bind": {"create_host_path": False, "propagation": "rprivate"}, + } + ] diff --git a/ix-dev/community/zerotier/templates/library/base_v2_1_0/validations.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/validations.py new file mode 100644 index 0000000000..43dd96b475 --- /dev/null +++ b/ix-dev/community/zerotier/templates/library/base_v2_1_0/validations.py @@ -0,0 +1,234 @@ +import re +import ipaddress + +try: + from .error import RenderError +except ImportError: + from error import RenderError + +OCTAL_MODE_REGEX = re.compile(r"^0[0-7]{3}$") + + +def valid_pull_policy_or_raise(pull_policy: str): + valid_policies = ("missing", "always", "never", "build") + if pull_policy not in valid_policies: + raise RenderError(f"Pull policy [{pull_policy}] is not valid. Valid options are: [{', '.join(valid_policies)}]") + return pull_policy + + +def valid_sysctl_or_raise(sysctl: str, host_network: bool): + if not sysctl: + raise RenderError("Sysctl cannot be empty") + if host_network and sysctl.startswith("net."): + raise RenderError(f"Sysctl [{sysctl}] cannot start with [net.] when host network is enabled") + + valid_sysctls = [ + "kernel.msgmax", + "kernel.msgmnb", + "kernel.msgmni", + "kernel.sem", + "kernel.shmall", + "kernel.shmmax", + "kernel.shmmni", + "kernel.shm_rmid_forced", + ] + # https://docs.docker.com/reference/cli/docker/container/run/#currently-supported-sysctls + if not sysctl.startswith("fs.mqueue.") and not sysctl.startswith("net.") and sysctl not in valid_sysctls: + raise RenderError( + f"Sysctl [{sysctl}] is not valid. Valid options are: [{', '.join(valid_sysctls)}], [net.*], [fs.mqueue.*]" + ) + return sysctl + + +def valid_redis_password_or_raise(password: str): + forbidden_chars = [" ", "'"] + for char in forbidden_chars: + if char in password: + raise RenderError(f"Redis password cannot contain [{char}]") + + +def valid_octal_mode_or_raise(mode: str): + mode = str(mode) + if not OCTAL_MODE_REGEX.match(mode): + raise RenderError(f"Expected [mode] to be a octal string, got [{mode}]") + return mode + + +def valid_host_path_propagation(propagation: str): + valid_propagations = ("shared", "slave", "private", "rshared", "rslave", "rprivate") + if propagation not in valid_propagations: + raise RenderError(f"Expected [propagation] to be one of [{', '.join(valid_propagations)}], got [{propagation}]") + return propagation + + +def valid_portal_scheme_or_raise(scheme: str): + schemes = ("http", "https") + if scheme not in schemes: + raise RenderError(f"Portal Scheme [{scheme}] is not valid. Valid options are: [{', '.join(schemes)}]") + return scheme + + +def valid_port_or_raise(port: int): + if port < 1 or port > 65535: + raise RenderError(f"Invalid port [{port}]. Valid ports are between 1 and 65535") + return port + + +def valid_ip_or_raise(ip: str): + try: + ipaddress.ip_address(ip) + except ValueError: + raise RenderError(f"Invalid IP address [{ip}]") + return ip + + +def valid_port_mode_or_raise(mode: str): + modes = ("ingress", "host") + if mode not in modes: + raise RenderError(f"Port Mode [{mode}] is not valid. Valid options are: [{', '.join(modes)}]") + return mode + + +def valid_port_protocol_or_raise(protocol: str): + protocols = ("tcp", "udp") + if protocol not in protocols: + raise RenderError(f"Port Protocol [{protocol}] is not valid. Valid options are: [{', '.join(protocols)}]") + return protocol + + +def valid_depend_condition_or_raise(condition: str): + valid_conditions = ("service_started", "service_healthy", "service_completed_successfully") + if condition not in valid_conditions: + raise RenderError( + f"Depend Condition [{condition}] is not valid. Valid options are: [{', '.join(valid_conditions)}]" + ) + return condition + + +def valid_cgroup_perm_or_raise(cgroup_perm: str): + valid_cgroup_perms = ("r", "w", "m", "rw", "rm", "wm", "rwm", "") + if cgroup_perm not in valid_cgroup_perms: + raise RenderError( + f"Cgroup Permission [{cgroup_perm}] is not valid. Valid options are: [{', '.join(valid_cgroup_perms)}]" + ) + return cgroup_perm + + +def allowed_dns_opt_or_raise(dns_opt: str): + disallowed_dns_opts = [] + if dns_opt in disallowed_dns_opts: + raise RenderError(f"DNS Option [{dns_opt}] is not allowed to added.") + return dns_opt + + +def valid_http_path_or_raise(path: str): + path = _valid_path_or_raise(path) + return path + + +def valid_fs_path_or_raise(path: str): + # There is no reason to allow / as a path, + # either on host or in a container side. + if path == "/": + raise RenderError(f"Path [{path}] cannot be [/]") + path = _valid_path_or_raise(path) + return path + + +def _valid_path_or_raise(path: str): + if path == "": + raise RenderError(f"Path [{path}] cannot be empty") + if not path.startswith("/"): + raise RenderError(f"Path [{path}] must start with /") + if "//" in path: + raise RenderError(f"Path [{path}] cannot contain [//]") + return path + + +def allowed_device_or_raise(path: str): + disallowed_devices = ["/dev/dri", "/dev/kfd", "/dev/bus/usb", "/dev/snd"] + if path in disallowed_devices: + raise RenderError(f"Device [{path}] is not allowed to be manually added.") + return path + + +def valid_network_mode_or_raise(mode: str, containers: list[str]): + valid_modes = ("host", "none") + if mode in valid_modes: + return mode + + if mode.startswith("service:"): + if mode[8:] not in containers: + raise RenderError(f"Service [{mode[8:]}] not found") + return mode + + raise RenderError( + f"Invalid network mode [{mode}]. Valid options are: [{', '.join(valid_modes)}] or [service:]" + ) + + +def valid_restart_policy_or_raise(policy: str, maximum_retry_count: int = 0): + valid_restart_policies = ("always", "on-failure", "unless-stopped", "no") + if policy not in valid_restart_policies: + raise RenderError( + f"Restart policy [{policy}] is not valid. Valid options are: [{', '.join(valid_restart_policies)}]" + ) + if policy != "on-failure" and maximum_retry_count != 0: + raise RenderError("Maximum retry count can only be set for [on-failure] restart policy") + + if maximum_retry_count < 0: + raise RenderError("Maximum retry count must be a positive integer") + + return policy + + +def valid_cap_or_raise(cap: str): + valid_policies = ( + "ALL", + "AUDIT_CONTROL", + "AUDIT_READ", + "AUDIT_WRITE", + "BLOCK_SUSPEND", + "BPF", + "CHECKPOINT_RESTORE", + "CHOWN", + "DAC_OVERRIDE", + "DAC_READ_SEARCH", + "FOWNER", + "FSETID", + "IPC_LOCK", + "IPC_OWNER", + "KILL", + "LEASE", + "LINUX_IMMUTABLE", + "MAC_ADMIN", + "MAC_OVERRIDE", + "MKNOD", + "NET_ADMIN", + "NET_BIND_SERVICE", + "NET_BROADCAST", + "NET_RAW", + "PERFMON", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_ADMIN", + "SYS_BOOT", + "SYS_CHROOT", + "SYS_MODULE", + "SYS_NICE", + "SYS_PACCT", + "SYS_PTRACE", + "SYS_RAWIO", + "SYS_RESOURCE", + "SYS_TIME", + "SYS_TTY_CONFIG", + "SYSLOG", + "WAKE_ALARM", + ) + + if cap not in valid_policies: + raise RenderError(f"Capability [{cap}] is not valid. " f"Valid options are: [{', '.join(valid_policies)}]") + + return cap diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/volume_mount.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/volume_mount.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/volume_mount.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/volume_mount.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/volume_mount_types.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/volume_mount_types.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/volume_mount_types.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/volume_mount_types.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/volume_sources.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/volume_sources.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/volume_sources.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/volume_sources.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/volume_types.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/volume_types.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/volume_types.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/volume_types.py diff --git a/ix-dev/community/zerotier/templates/library/base_v2_0_25/volumes.py b/ix-dev/community/zerotier/templates/library/base_v2_1_0/volumes.py similarity index 100% rename from ix-dev/community/zerotier/templates/library/base_v2_0_25/volumes.py rename to ix-dev/community/zerotier/templates/library/base_v2_1_0/volumes.py